diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index b1ef52818..5cd74e363 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -18,6 +18,14 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev + + + - name: Cache Android NDK + id: cache-android-ndk + uses: actions/cache@v4 + with: + path: android-ndk-r25b + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-cache-android-ndk-${{ hashFiles('**/assembly/android/build-android-tonlib.sh') }} - name: Build TON run: | diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index d464d8a2a..c8147677e 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -12,17 +12,44 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 + mkdir ~/.ccache 3pp - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache - name: Build TON run: | @@ -30,7 +57,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -41,12 +75,18 @@ jobs: ./create-appimages.sh aarch64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 6433df0b5..540fffece 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -16,17 +16,39 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os != 'ubuntu-24.04-arm' name: Install llvm-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -34,10 +56,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 4f78ece9d..cb02b7bc2 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -12,23 +12,43 @@ jobs: with: submodules: 'recursive' - - name: Install system libraries + - name: Date Stamp + shell: bash + id: date-stamp run: | - sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev - sudo apt remove libgsl-dev + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" - - name: Install gcc-11 g++-11 + - name: Install system libraries run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 + sudo apt update + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 + mkdir ~/.ccache 3pp - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache - name: Build TON run: | @@ -36,7 +56,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -47,12 +74,18 @@ jobs: ./create-appimages.sh x86_64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index bf78f7df1..ed05d18aa 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.os }} steps: @@ -16,22 +16,37 @@ jobs: with: submodules: 'recursive' - - name: Install system libraries + - name: Date Stamp + shell: bash + id: date-stamp run: | - sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" - - if: matrix.os == 'ubuntu-20.04' + - name: Install system libraries run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os != 'ubuntu-24.04' run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Restore cache TON + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -39,10 +54,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml index 5e50a4680..6a6a6a255 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-13-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 13.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml index 8eb3af70a..995ff1c40 100644 --- a/.github/workflows/build-ton-macos-14-arm64-portable.yml +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-14-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 14.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 00d8f6392..a1e0782b7 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 16.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-15 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index 7481f9ff6..5ef9266af 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 14.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-14 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index 6a69b2e35..6200af663 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-13 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index bac0cf984..f9f861148 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -16,6 +16,27 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + mkdir -p 3pp_emscripten + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} + + - name: Cache 3pp Emscripten + id: cache-3pp-emscripten + uses: actions/cache@v4 + with: + path: 3pp_emscripten + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp_em-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} - name: Build TON WASM artifacts run: | @@ -43,7 +64,6 @@ jobs: cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js npx func-js stdlib.fc intrinsics.fc --fift ./output.f - - name: Upload artifacts uses: actions/upload-artifact@master with: diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml index fb8438a12..4cd30ef6f 100644 --- a/.github/workflows/create-tolk-release.yml +++ b/.github/workflows/create-tolk-release.yml @@ -9,6 +9,9 @@ on: permissions: write-all +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: create-release: runs-on: ubuntu-22.04 @@ -152,3 +155,11 @@ jobs: asset_name: ton-wasm.zip tag: ${{ inputs.tag }} + - name: Upload stdlib + run: | + mkdir smartcont_lib + cd smartcont_lib + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + zip -r smartcont_lib.zip . + gh release upload ${{ inputs.tag }} smartcont_lib.zip + diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index 00aa5015e..bd97ed233 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -11,21 +11,26 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 with: submodules: 'recursive' + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU uses: docker/setup-qemu-action@v3.5.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.10.0 with: - driver-opts: image=moby/buildkit:v0.11.0 + driver-opts: image=moby/buildkit:v0.20.2 - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -34,28 +39,80 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Get tag as branch name id: tag run: | echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + with: + driver-opts: image=moby/buildkit:v0.20.2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index aa4eaeef5..7a5eee10a 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -5,14 +5,13 @@ on: push: branches: - 'master' - env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 @@ -32,16 +31,47 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Get next tag id: tag @@ -57,12 +87,52 @@ jobs: fi - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index baaad778b..8de9e11be 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -9,7 +9,7 @@ defaults: jobs: build: - runs-on: windows-2019 + runs-on: windows-2022 steps: - name: Get Current OS version @@ -21,13 +21,30 @@ jobs: with: submodules: 'recursive' + - name: Create directories + run: | + mkdir third_libs + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: third_libs + key: ${{ runner.os }}-${{ runner.arch }}-windows-2022-3pp-${{ hashFiles('**/assembly/native/build-windows-2022.bat') }} + - name: Build TON run: | git submodule sync --recursive git submodule update - copy assembly\native\build-windows-github-2019.bat . - copy assembly\native\build-windows-2019.bat . - build-windows-github-2019.bat Enterprise + copy assembly\native\build-windows-github-2022.bat . + copy assembly\native\build-windows-2022.bat . + build-windows-github-2022.bat Enterprise + ccache -sp + +# - name: Run Tests +# run: | +# cd build +# ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.gitignore b/.gitignore index e5bb366ce..986ee6daf 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ crypto/smartcont/auto/ test/regression-tests.cache/ *.swp **/*build*/ +.cache .idea .vscode .DS_Store @@ -24,3 +25,4 @@ libsodium-1.0.18-stable-msvc.zip libmicrohttpd-0.9.77-w32-bin.zip openssl-3.1.4.zip readline-5.0-1-lib.zip +libtunnel.a \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index cea3fc7ec..16aaf1461 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -29,60 +29,12 @@ if (TON_REAL_BINARY_DIR STREQUAL TON_REAL_SOURCE_DIR) message(FATAL_ERROR "In-source build failed.") endif() -# HAVE_SSE42 for crc32c and rocksdb -include(CheckCXXSourceCompiles) -# Check for SSE4.2 support in the compiler. -set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS}) -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /arch:AVX") -else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -msse4.2") -endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") -check_cxx_source_compiles(" -#if defined(_MSC_VER) -#include -#else // !defined(_MSC_VER) -#include -#include -#endif // defined(_MSC_VER) - -int main() { - _mm_crc32_u8(0, 0); _mm_crc32_u32(0, 0); -#if defined(_M_X64) || defined(__x86_64__) - _mm_crc32_u64(0, 0); -#endif // defined(_M_X64) || defined(__x86_64__) - return 0; -} -" CRC32C_HAVE_SSE42) -set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS}) - -if(NOT MSVC) - set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul") -endif() -CHECK_CXX_SOURCE_COMPILES(" -#include -#include -#include -int main() { - volatile uint32_t x = _mm_crc32_u32(0, 0); - const auto a = _mm_set_epi64x(0, 0); - const auto b = _mm_set_epi64x(0, 0); - const auto c = _mm_clmulepi64_si128(a, b, 0x00); - auto d = _mm_cvtsi128_si64(c); -} -" ROCKSDB_HAVE_SSE42) -unset(CMAKE_REQUIRED_FLAGS) - -if (ROCKSDB_HAVE_SSE42 AND CRC32C_HAVE_SSE42) - set(HAVE_SSE42 TRUE) -else() - set(HAVE_SSE42 FALSE) -endif() - set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) +set(CMAKE_COLOR_DIAGNOSTICS TRUE) + #BEGIN internal option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) @@ -105,6 +57,31 @@ option(TON_USE_ASAN "Use \"ON\" to enable AddressSanitizer." OFF) option(TON_USE_TSAN "Use \"ON\" to enable ThreadSanitizer." OFF) option(TON_USE_UBSAN "Use \"ON\" to enable UndefinedBehaviorSanitizer." OFF) set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") +option(TON_USE_GO_TUNNEL "Use \"ON\" to enable ADNL Tunnel over shared Go library." OFF) + +if (TON_USE_GO_TUNNEL) + set(TUNNEL_GO_LIB_PATH "${CMAKE_CURRENT_SOURCE_DIR}/libtunnel.a") + if (EXISTS "${TUNNEL_GO_LIB_PATH}") + message(STATUS "Found ADNL Tunnel library (Go): ${TUNNEL_GO_LIB_PATH}") + + add_library(tunnel STATIC IMPORTED) + set_target_properties(tunnel PROPERTIES IMPORTED_LOCATION "${TUNNEL_GO_LIB_PATH}") + set(TUNNEL_LIB_IF_USED "tunnel") + add_compile_definitions(TON_USE_GO_TUNNEL) + else() + message(FATAL_ERROR "Missing ADNL Tunnel library (Go), but enabled: ${TUNNEL_GO_LIB_PATH}") + endif() +endif() + +option(TON_PRINT_BACKTRACE_ON_CRASH "Attempt to print a backtrace when a fatal signal is caught" ON) + +option(TON_USE_LLD "Use LLD for linking" OFF) + +if (TON_USE_LLD) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") +endif() #BEGIN M1 support EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE ) @@ -127,9 +104,12 @@ endif() if (WIN32) message("Add wingetopt") - add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + function(wingetopt_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + endfunction() + wingetopt_scope() set(WINGETOPT_FOUND 1) - message(STATUS "Use wingetopt") endif() set(CRC32C_BUILD_TESTS OFF CACHE BOOL "Build CRC32C's unit tests") @@ -137,16 +117,15 @@ set(CRC32C_BUILD_BENCHMARKS OFF CACHE BOOL "Build CRC32C's benchmarks") set(CRC32C_USE_GLOG OFF CACHE BOOL "Build CRC32C's tests with Google Logging") set(CRC32C_INSTALL OFF CACHE BOOL "Install CRC32C's header and library") message("Add crc32c") -if (NOT MSVC) - set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) - # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") - add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) - set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) - unset(OLD_CMAKE_CXX_FLAGS) -else() +function(crc32_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + if (NOT MSVC) + # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") + endif() add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) -endif() +endfunction() +crc32_scope() set(CRC32C_FOUND 1) if (TON_USE_ROCKSDB) @@ -156,9 +135,13 @@ if (TON_USE_ROCKSDB) set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") + set(USE_RTTI ON CACHE BOOL "use rtti") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) + # Broken CMake in rocksdb alters properties it has no business changing. + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK) endif() option(USE_COROUTINES "experimental support of coroutines" OFF) @@ -188,11 +171,12 @@ include(BuildSECP256K1) # Configure CCache if available find_program(CCACHE_FOUND ccache) -#set(CCACHE_FOUND 0) if (CCACHE_FOUND) - message(STATUS "Found ccache") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) + if (NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) + message(STATUS "Using ccache") + set(CMAKE_C_COMPILER_LAUNCHER ccache) + set(CMAKE_CXX_COMPILER_LAUNCHER ccache) + endif() else() message(STATUS "Could NOT find ccache") endif() @@ -224,7 +208,7 @@ endif() if (TON_ARCH AND NOT MSVC) CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) if (TON_ARCH STREQUAL "apple-m1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") elseif(COMPILER_OPT_ARCH_SUPPORTED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}") elseif(NOT TON_ARCH STREQUAL "native") @@ -341,6 +325,14 @@ add_cxx_compiler_flag("-Qunused-arguments") add_cxx_compiler_flag("-Wno-unused-private-field") add_cxx_compiler_flag("-Wno-redundant-move") +if (GCC OR CLANG) + if (CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + # For historical reasons, CMake falls back to -O2 optimization level when CMAKE_BUILD_TYPE is + # set to RelWithDebInfo. + add_compile_options(-O3) + endif() +endif() + #add_cxx_compiler_flag("-Wno-unused-function") #add_cxx_compiler_flag("-Wno-unused-variable") #add_cxx_compiler_flag("-Wno-shorten-64-to-32") @@ -351,22 +343,13 @@ if (CLANG) #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") endif() if (TON_USE_ASAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=address") add_definitions(-DTD_USE_ASAN=1) endif() if (TON_USE_TSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=thread") endif() if (TON_USE_UBSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=undefined") endif() #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") @@ -578,7 +561,6 @@ endif() enable_testing() set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests.ans --filter -Bench") separate_arguments(TEST_OPTIONS) -add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) diff --git a/Changelog.md b/Changelog.md index 4dce39fcf..b104d10ad 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,31 @@ +## 2025.07 Accelerator Update + +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) + +## 2025.06 Update + +1. ADNL and candidate broadcast optimization +2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. +3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). +4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic +5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods +6. Optimized package import for archive nodes + +Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). + + +## 2025.04 Update + +1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. +2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. +3. [TVM version v10](./doc/GlobalVersions.md) +4. Overlay broadcast speed up and improved network stats. +5. Fixed some issues in tonlib +6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) +7. Fix SDBEGINS(Q) in Asm.fif + +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes); @ice-charon (tonlib fixes); RSquad team (due payments improvements in v10); Arayz, Robinlzw, @wy666444 @Lucian-code233 from TonBit (improvements in RUNVM); @Skydev0h and @pyAndr3w (Asm.fif). + ## 2025.03 Update 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) 2. Optmization of validation process, in particular CellStorageStat. diff --git a/Dockerfile b/Dockerfile index f1b836bf9..7a53500db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN mkdir build && \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \ generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \ - blockchain-explorer emulator tonlibjson http-proxy adnl-proxy + blockchain-explorer emulator tonlibjson http-proxy adnl-proxy dht-ping-servers dht-resolve FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive @@ -50,6 +50,8 @@ COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/loca COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/ COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/ COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-ping-servers /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-resolve /usr/local/bin/ COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/ COPY --from=builder /ton/build/http/http-proxy /usr/local/bin/ COPY --from=builder /ton/build/adnl/adnl-proxy /usr/local/bin/ diff --git a/README.md b/README.md index 897ba809a..8376f4860 100644 --- a/README.md +++ b/README.md @@ -149,3 +149,21 @@ Linux and MacOS binaries are available for both x86-64 and arm64 architectures. ## Running tests Tests are executed by running `ctest` in the build directory. See `doc/Tests.md` for more information. + +## Using ADNL tunnel + +### At node compilation +1. Clone https://github.com/ton-blockchain/adnl-tunnel and install golang 1.23.3 or newer + * `cd adnl-tunnel` + * `make library` + * It will build `libtunnel.a` +2. Copy `libtunnel.a` to ton src directory root (usually `/usr/src/ton`). +3. On the first step of ton node compilation run cmake with option `-DTON_USE_GO_TUNNEL=ON` to enable tunnel. +4. Build ton node as usual. + +### At node startup +1. Run validator-engine with `--tunnel-config /path/to/tunnel-config.json` startup argument. +2. It will create example tunnel config file at specified path (`/path/to/tunnel-config.json`). +3. Fill it with desired tunnel configuration, enable payments and top up wallet address if needed. +4. Run validator-engine `--tunnel-config /path/to/tunnel-config.json` again, and follow instructions in console if any. +5. When setup is completed and node started, you can stop it and run in daemon mode as usual. \ No newline at end of file diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index 111c4c500..3604dfb3a 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #BEGIN internal if (NOT TON_ONLY_TONLIB) set(ADNL_HEADERS @@ -25,8 +23,6 @@ set(ADNL_HEADERS adnl-peer.h adnl-peer.hpp adnl-query.h - adnl-static-nodes.h - adnl-static-nodes.hpp adnl-proxy-types.h adnl-proxy-types.hpp adnl.h @@ -48,7 +44,6 @@ set(ADNL_SOURCE adnl-peer.cpp adnl-query.cpp adnl-channel.cpp - adnl-static-nodes.cpp adnl-proxy-types.cpp utils.cpp ${ADNL_HEADERS} diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index 9602b521e..873d34b11 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -31,6 +31,14 @@ void AdnlExtClientImpl::alarm() { next_create_at_ = td::Timestamp::in(10.0); alarm_timestamp() = next_create_at_; + if (!dst_host_.empty()) { + auto S = dst_addr_.init_host_port(dst_host_); + if (S.is_error()) { + LOG(INFO) << "failed to connect to " << dst_host_ << ": " << S; + return; + } + LOG(DEBUG) << "resolved " << dst_host_ << " -> " << dst_addr_; + } auto fd = td::SocketFd::open(dst_addr_); if (fd.is_error()) { LOG(INFO) << "failed to connect to " << dst_addr_ << ": " << fd.move_as_error(); @@ -166,6 +174,12 @@ td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, td: return td::actor::create_actor("extclient", std::move(dst), dst_addr, std::move(callback)); } +td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback) { + return td::actor::create_actor("extclient", std::move(dst), std::move(dst_host), + std::move(callback)); +} + td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) { diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index b9a5d570a..9e537a5a5 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -39,6 +39,8 @@ class AdnlExtClient : public td::actor::Actor { td::Promise promise) = 0; static td::actor::ActorOwn create(AdnlNodeIdFull dst, td::IPAddress dst_addr, std::unique_ptr callback); + static td::actor::ActorOwn create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback); static td::actor::ActorOwn create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback); }; diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 1dd7d2ba3..76de83ab4 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -71,6 +71,9 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlExtClientImpl(AdnlNodeIdFull dst_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), dst_addr_(dst_addr), callback_(std::move(callback)) { } + AdnlExtClientImpl(AdnlNodeIdFull dst_id, std::string dst_host, std::unique_ptr callback) + : dst_(std::move(dst_id)), dst_host_(std::move(dst_host)), callback_(std::move(callback)) { + } AdnlExtClientImpl(AdnlNodeIdFull dst_id, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), local_id_(local_id), dst_addr_(dst_addr), callback_(std::move(callback)) { @@ -133,6 +136,7 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlNodeIdFull dst_; PrivateKey local_id_; td::IPAddress dst_addr_; + std::string dst_host_; std::unique_ptr callback_; diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 162a53afb..627a25cb5 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -31,7 +31,7 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) { td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = f->query_id_](td::Result R) { if (R.is_error()) { auto S = R.move_as_error(); - LOG(WARNING) << "failed ext query: " << S; + LOG(INFO) << "failed ext query: " << S; } else { auto B = create_tl_object(query_id, R.move_as_ok()); td::actor::send_closure(SelfId, &AdnlInboundConnection::send, serialize_tl_object(B, true)); diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index 077fb9390..6d748e34a 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -76,6 +76,46 @@ size_t AdnlNetworkManagerImpl::add_listening_udp_port(td::uint16 port) { return idx; } +constexpr int TUNNEL_FAKE_PORT = 0; + +size_t AdnlNetworkManagerImpl::add_tunnel_udp_port(std::string global_config, std::string tunnel_config, td::Promise on_ready, + td::actor::Scheduler *scheduler) { + auto it = port_2_socket_.find(TUNNEL_FAKE_PORT); + if (it != port_2_socket_.end()) { + return it->second; + } + class Callback : public td::UdpServer::TunnelCallback { + public: + Callback(td::actor::ActorShared manager, size_t idx, td::actor::Scheduler *scheduler, TunnelEventsHandler* tunnel_events_handler) + : manager_(std::move(manager)), idx_(idx), scheduler_(scheduler), tunnel_events_handler_(tunnel_events_handler) { + } + + private: + TunnelEventsHandler* tunnel_events_handler_; + td::actor::ActorShared manager_; + size_t idx_; + td::actor::Scheduler *scheduler_; + void on_udp_message(td::UdpMessage udp_message) override { + scheduler_->run_in_context_external([&] { + td::actor::send_closure_later(manager_, &AdnlNetworkManagerImpl::receive_udp_message, std::move(udp_message), + idx_); + }); + } + void on_in_addr_update(const td::IPAddress ip) override { + tunnel_events_handler_->on_in_addr_update(ip); + } + }; + + auto idx = udp_sockets_.size(); + auto X = td::UdpServer::create_via_tunnel("udp tunnel server", global_config, tunnel_config, + std::make_unique(actor_shared(this), idx, scheduler, tunnel_events_handler_.get()), + std::move(on_ready)); + X.ensure(); + port_2_socket_[TUNNEL_FAKE_PORT] = idx; + udp_sockets_.push_back(UdpSocketDesc{TUNNEL_FAKE_PORT, X.move_as_ok()}); + return idx; +} + void AdnlNetworkManagerImpl::add_self_addr(td::IPAddress addr, AdnlCategoryMask cat_mask, td::uint32 priority) { auto port = td::narrow_cast(addr.get_port()); size_t idx = add_listening_udp_port(port); @@ -92,6 +132,23 @@ void AdnlNetworkManagerImpl::add_self_addr(td::IPAddress addr, AdnlCategoryMask out_desc_[priority].push_back(std::move(d)); } +void AdnlNetworkManagerImpl::add_tunnel(std::string global_config, std::string tunnel_config, AdnlCategoryMask cat_mask, td::uint32 priority, + td::Promise on_ready, td::actor::Scheduler *scheduler) { + size_t idx = add_tunnel_udp_port(global_config, tunnel_config, std::move(on_ready), scheduler); + + add_in_addr(InDesc{TUNNEL_FAKE_PORT, nullptr, cat_mask}, idx); + auto d = OutDesc{TUNNEL_FAKE_PORT, td::IPAddress{}, nullptr, idx}; + for (auto &it : out_desc_[priority]) { + if (it == d) { + it.cat_mask |= cat_mask; + return; + } + } + + d.cat_mask = cat_mask; + out_desc_[priority].push_back(std::move(d)); +} + void AdnlNetworkManagerImpl::add_proxy_addr(td::IPAddress addr, td::uint16 local_port, std::shared_ptr proxy, AdnlCategoryMask cat_mask, td::uint32 priority) { size_t idx = add_listening_udp_port(local_port); diff --git a/adnl/adnl-network-manager.h b/adnl/adnl-network-manager.h index 67cf602aa..d94d53372 100644 --- a/adnl/adnl-network-manager.h +++ b/adnl/adnl-network-manager.h @@ -59,17 +59,25 @@ class AdnlNetworkManager : public td::actor::Actor { public: //using ConnHandle = td::uint64; class Callback { - public: + public: virtual ~Callback() = default; //virtual void receive_packet(td::IPAddress addr, ConnHandle conn_handle, td::BufferSlice data) = 0; virtual void receive_packet(td::IPAddress addr, AdnlCategoryMask cat_mask, td::BufferSlice data) = 0; }; + class TunnelEventsHandler { + public: + virtual ~TunnelEventsHandler() = default; + virtual void on_in_addr_update(td::IPAddress ip) = 0; + }; static td::actor::ActorOwn create(td::uint16 out_port); virtual ~AdnlNetworkManager() = default; virtual void install_callback(std::unique_ptr callback) = 0; + virtual void install_tunnel_events_handler(std::unique_ptr handler) = 0; + virtual void add_tunnel(std::string global_config, std::string tunnel_config, AdnlCategoryMask cat_mask, td::uint32 priority, + td::Promise on_ready, td::actor::Scheduler *scheduler) = 0; virtual void add_self_addr(td::IPAddress addr, AdnlCategoryMask cat_mask, td::uint32 priority) = 0; virtual void add_proxy_addr(td::IPAddress addr, td::uint16 local_port, std::shared_ptr proxy, AdnlCategoryMask cat_mask, td::uint32 priority) = 0; diff --git a/adnl/adnl-network-manager.hpp b/adnl/adnl-network-manager.hpp index a77be19d6..4f7f42123 100644 --- a/adnl/adnl-network-manager.hpp +++ b/adnl/adnl-network-manager.hpp @@ -95,6 +95,10 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { size_t in_desc{std::numeric_limits::max()}; bool allow_proxy{false}; }; + struct TunnelDesc { + size_t index{}; + td::IPAddress address; + }; OutDesc *choose_out_iface(td::uint8 cat, td::uint32 priority); @@ -105,6 +109,10 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { callback_ = std::move(callback); } + void install_tunnel_events_handler(std::unique_ptr handler) override { + tunnel_events_handler_ = std::move(handler); + } + void alarm() override; void start_up() override { alarm_timestamp() = td::Timestamp::in(60.0); @@ -127,6 +135,8 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { in_desc_.push_back(std::move(desc)); } + void add_tunnel(std::string global_config, std::string tunnel_config, AdnlCategoryMask cat_mask, td::uint32 priority, td::Promise on_ready, + td::actor::Scheduler *scheduler) override; void add_self_addr(td::IPAddress addr, AdnlCategoryMask cat_mask, td::uint32 priority) override; void add_proxy_addr(td::IPAddress addr, td::uint16 local_port, std::shared_ptr proxy, AdnlCategoryMask cat_mask, td::uint32 priority) override; @@ -141,12 +151,14 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { } } + size_t add_tunnel_udp_port(std::string global_config, std::string tunnel_config, td::Promise on_ready, td::actor::Scheduler *scheduler); size_t add_listening_udp_port(td::uint16 port); void receive_udp_message(td::UdpMessage message, size_t idx); void proxy_register(OutDesc &desc); private: std::unique_ptr callback_; + std::unique_ptr tunnel_events_handler_; std::map> out_desc_; std::vector in_desc_; diff --git a/adnl/adnl-node-id.hpp b/adnl/adnl-node-id.hpp index 2d3ade164..43e3629f4 100644 --- a/adnl/adnl-node-id.hpp +++ b/adnl/adnl-node-id.hpp @@ -37,7 +37,9 @@ class AdnlNodeIdShort { } explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) { } - explicit AdnlNodeIdShort(tl_object_ptr obj) : hash_(obj->id_) { + explicit AdnlNodeIdShort(tl_object_ptr &&obj) : hash_(obj->id_) { + } + explicit AdnlNodeIdShort(const tl_object_ptr &obj) : hash_(obj->id_) { } const auto &pubkey_hash() const { diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index b8aab5671..5a92ca1c1 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -84,6 +84,32 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ << " (len=" << (data.size() + 32) << ")"; } +void AdnlPeerTableImpl::update_id(AdnlPeerTableImpl::PeerInfo &peer_info, AdnlNodeIdFull &&peer_id) { + if (peer_info.peer_id.empty()) { + peer_info.peer_id = std::move(peer_id); + for (auto &e: peer_info.peers) { + td::actor::send_closure(e.second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } +} + +td::actor::ActorOwn &AdnlPeerTableImpl::get_peer_pair( + AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, + AdnlNodeIdShort local_id, AdnlPeerTableImpl::LocalIdInfo &local_id_info) { + auto it = peer_info.peers.find(local_id); + if (it == peer_info.peers.end()) { + it = peer_info.peers.emplace(local_id, + AdnlPeerPair::create(network_manager_, actor_id(this), + local_id_info.mode, local_id_info.local_id.get(), + dht_node_, local_id, peer_id)) + .first; + if (!peer_info.peer_id.empty()) { + td::actor::send_closure(it->second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } + return it->second; +} + void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { packet.run_basic_checks().ensure(); @@ -91,8 +117,9 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket VLOG(ADNL_INFO) << this << ": dropping IN message [?->" << dst << "]: destination not set"; return; } + AdnlNodeIdShort src = packet.from_short(); - auto it = peers_.find(packet.from_short()); + auto it = peers_.find(src); if (it == peers_.end()) { if (!packet.inited_from()) { VLOG(ADNL_NOTICE) << this << ": dropping IN message [" << packet.from_short() << "->" << dst @@ -105,11 +132,7 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket return; } - it = peers_ - .emplace(packet.from_short(), - AdnlPeer::create(network_manager_, actor_id(this), dht_node_, packet.from_short())) - .first; - CHECK(it != peers_.end()); + it = peers_.try_emplace(src).first; } auto it2 = local_ids_.find(dst); @@ -118,8 +141,13 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket << "]: unknown dst (but how did we decrypt message?)"; return; } - td::actor::send_closure(it->second, &AdnlPeer::receive_packet, dst, it2->second.mode, it2->second.local_id.get(), - std::move(packet), serialized_size); + + if (packet.inited_from()) { + update_id(it->second, packet.from()); + } + + td::actor::send_closure(get_peer_pair(src, it->second, dst, it2->second), + &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -129,31 +157,25 @@ void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, Ad auto it2 = local_ids_.find(local_id); CHECK(it2 != local_ids_.end()); - auto it = peers_.find(id_short); - if (it == peers_.end()) { - it = peers_.emplace(id_short, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, id_short)).first; - CHECK(it != peers_.end()); - } - td::actor::send_closure(it->second, &AdnlPeer::update_id, std::move(id)); + auto &peer_info = peers_[id_short]; + update_id(peer_info, std::move(id)); if (!addr_list.empty()) { - td::actor::send_closure(it->second, &AdnlPeer::update_addr_list, local_id, it2->second.mode, - it2->second.local_id.get(), std::move(addr_list)); + td::actor::send_closure(get_peer_pair(id_short, peer_info, local_id, it2->second), + &AdnlPeerPair::update_addr_list, std::move(addr_list)); } } void AdnlPeerTableImpl::add_static_nodes_from_config(AdnlNodesList nodes) { - for (auto &it : nodes.nodes()) { - add_static_node(it); + for (auto &node : nodes.nodes()) { + auto id_short = node.compute_short_id(); + VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; + static_nodes_.emplace(id_short, std::move(node)); } } void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) { - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -161,8 +183,10 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst return; } - td::actor::send_closure(it->second, &AdnlPeer::send_one_message, src, it2->second.mode, it2->second.local_id.get(), - OutboundAdnlMessage{std::move(message), flags}); + std::vector messages; + messages.push_back(OutboundAdnlMessage{std::move(message), flags}); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_messages, std::move(messages)); } void AdnlPeerTableImpl::answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, @@ -182,11 +206,7 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std VLOG(ADNL_WARNING) << "DUMP: " << td::buffer_to_hex(data.as_slice().truncate(128)); return; } - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -194,8 +214,8 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std return; } - td::actor::send_closure(it->second, &AdnlPeer::send_query, src, it2->second.mode, it2->second.local_id.get(), name, - std::move(promise), timeout, std::move(data), 0); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), 0); } void AdnlPeerTableImpl::add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) { @@ -247,8 +267,10 @@ void AdnlPeerTableImpl::unsubscribe(AdnlNodeIdShort dst, std::string prefix) { void AdnlPeerTableImpl::register_dht_node(td::actor::ActorId dht_node) { dht_node_ = dht_node; - for (auto &peer : peers_) { - td::actor::send_closure(peer.second, &AdnlPeer::update_dht_node, dht_node_); + for (auto &e : peers_) { + for (auto &e2 : e.second.peers) { + td::actor::send_closure(e2.second, &AdnlPeerPair::update_dht_node, dht_node_); + } } for (auto &local_id : local_ids_) { td::actor::send_closure(local_id.second.local_id, &AdnlLocalId::update_dht_node, dht_node_); @@ -332,8 +354,6 @@ void AdnlPeerTableImpl::get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNode AdnlPeerTableImpl::AdnlPeerTableImpl(std::string db_root, td::actor::ActorId keyring) { keyring_ = keyring; - static_nodes_manager_ = AdnlStaticNodesManager::create(); - if (!db_root.empty()) { db_ = AdnlDb::create(db_root + "/adnl"); } @@ -382,7 +402,64 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ promise.set_value("undefined"); return; } - td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); + auto it2 = it->second.peers.find(l_id); + if (it2 == it->second.peers.end()) { + promise.set_value("undefined"); + return; + } + td::actor::send_closure(it2->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); +} + +void AdnlPeerTableImpl::get_stats_peer(AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, bool all, + td::Promise>> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise>> promise) + : promise_(std::move(promise)) { + } + + void got_peer_pair_stats(tl_object_ptr peer_pair) { + if (peer_pair) { + result_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + td::Promise>> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); + + for (auto &[local_id, peer_pair] : peer_info.peers) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, + [local_id = local_id, peer_id = peer_id, + callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id + << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); } void AdnlPeerTableImpl::get_stats(bool all, td::Promise> promise) { @@ -453,8 +530,8 @@ void AdnlPeerTableImpl::get_stats(bool all, td::Promise>> R) { if (R.is_error()) { VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 055f32ac1..13b4adb53 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -97,9 +97,7 @@ class AdnlPeerTable : public Adnl { td::actor::ActorId channel) = 0; virtual void unregister_channel(AdnlChannelIdShort id) = 0; - virtual void add_static_node(AdnlNode node) = 0; - virtual void del_static_node(AdnlNodeIdShort id) = 0; - virtual void get_static_node(AdnlNodeIdShort id, td::Promise promise) = 0; + virtual td::Result get_static_node(AdnlNodeIdShort id) = 0; virtual void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node, td::Promise promise) = 0; diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 9ad61b653..408f846d0 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -28,7 +28,6 @@ #include "adnl-local-id.h" #include "adnl-query.h" #include "utils.hpp" -#include "adnl-static-nodes.h" #include "adnl-ext-server.h" #include "adnl-address-list.h" @@ -86,15 +85,12 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, td::Promise promise) override; - void add_static_node(AdnlNode node) override { - CHECK(!static_nodes_manager_.empty()); - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::add_node, std::move(node)); - } - void del_static_node(AdnlNodeIdShort id) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::del_node, id); - } - void get_static_node(AdnlNodeIdShort id, td::Promise promise) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::get_node, id, std::move(promise)); + td::Result get_static_node(AdnlNodeIdShort id) override { + auto it = static_nodes_.find(id); + if (it == static_nodes_.end()) { + return td::Status::Error(ErrorCode::notready, "static node not found"); + } + return it->second; } void deliver(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override; void deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data, @@ -116,20 +112,26 @@ class AdnlPeerTableImpl : public AdnlPeerTable { } private: + struct PeerInfo { + AdnlNodeIdFull peer_id; + std::map> peers; + }; + struct LocalIdInfo { td::actor::ActorOwn local_id; td::uint8 cat; td::uint32 mode; }; + td::actor::ActorId keyring_; td::actor::ActorId network_manager_; td::actor::ActorId dht_node_; - td::actor::ActorOwn static_nodes_manager_; + std::map static_nodes_; void deliver_one_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message); - std::map> peers_; + std::map peers_; std::map local_ids_; std::map, td::uint8>> channels_; @@ -140,6 +142,11 @@ class AdnlPeerTableImpl : public AdnlPeerTable { AdnlNodeIdShort proxy_addr_; //std::map> out_queries_; //td::uint64 last_query_id_ = 1; + + static void update_id(PeerInfo &peer_info, AdnlNodeIdFull &&peer_id); + td::actor::ActorOwn &get_peer_pair(AdnlNodeIdShort peer_id, PeerInfo &peer_info, AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); + static void get_stats_peer(AdnlNodeIdShort peer_id, PeerInfo &peer_info, bool all, + td::Promise>> promise); }; inline td::StringBuilder &operator<<(td::StringBuilder &sb, const AdnlPeerTableImpl::PrintId &id) { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index ab4600581..4913216ee 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -475,13 +475,12 @@ void AdnlPeerPairImpl::alarm_query(AdnlQueryId id) { AdnlPeerPairImpl::AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, - td::actor::ActorId local_actor, td::actor::ActorId peer, + td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { network_manager_ = network_manager; peer_table_ = peer_table; local_actor_ = local_actor; - peer_ = peer; dht_node_ = dht_node; mode_ = local_mode; @@ -598,11 +597,11 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { - VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; + VLOG(ADNL_INFO) << this << ": dropping too big huge message: size=" << size; return; } if (message.hash().is_zero()) { - VLOG(ADNL_WARNING) << this << ": dropping huge message with zero hash"; + VLOG(ADNL_INFO) << this << ": dropping huge message with zero hash"; return; } if (message.hash() != huge_message_hash_) { @@ -871,19 +870,6 @@ void AdnlPeerPairImpl::get_stats(bool all, td::Promise peer, td::actor::ActorId network_manager, td::actor::ActorId adnl) { @@ -902,171 +888,13 @@ void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { td::actor::ActorOwn AdnlPeerPair::create( td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId peer_actor, + td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { auto X = td::actor::create_actor("peerpair", network_manager, peer_table, local_mode, local_actor, - peer_actor, dht_node, local_id, peer_id); + dht_node, local_id, peer_id); return td::actor::ActorOwn(std::move(X)); } -td::actor::ActorOwn AdnlPeer::create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) { - auto X = td::actor::create_actor("peer", network_manager, peer_table, dht_node, peer_id); - return td::actor::ActorOwn(std::move(X)); -} - -void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) { - if (packet.inited_from()) { - update_id(packet.from()); - } - - auto it = peer_pairs_.find(dst); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, dst_mode, dst_actor, actor_id(this), dht_node_, dst, - peer_id_short_); - peer_pairs_.emplace(dst, std::move(X)); - it = peer_pairs_.find(dst); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); -} - -void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_messages, std::move(messages)); -} - -void AdnlPeerImpl::send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), - flags); -} - -void AdnlPeerImpl::del_local_id(AdnlNodeIdShort local_id) { - peer_pairs_.erase(local_id); -} - -void AdnlPeerImpl::update_dht_node(td::actor::ActorId dht_node) { - dht_node_ = dht_node; - for (auto it = peer_pairs_.begin(); it != peer_pairs_.end(); it++) { - td::actor::send_closure(it->second, &AdnlPeerPair::update_dht_node, dht_node_); - } -} - -void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) { - auto it = peer_pairs_.find(l_id); - if (it == peer_pairs_.end()) { - promise.set_value("undefined"); - return; - } - - td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); -} - -void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) { - auto it = peer_pairs_.find(local_id); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, local_mode, local_actor, actor_id(this), dht_node_, - local_id, peer_id_short_); - peer_pairs_.emplace(local_id, std::move(X)); - it = peer_pairs_.find(local_id); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list)); -} - -void AdnlPeerImpl::get_stats(bool all, td::Promise>> promise) { - class Cb : public td::actor::Actor { - public: - explicit Cb(td::Promise>> promise) - : promise_(std::move(promise)) { - } - - void got_peer_pair_stats(tl_object_ptr peer_pair) { - if (peer_pair) { - result_.push_back(std::move(peer_pair)); - } - dec_pending(); - } - - void inc_pending() { - ++pending_; - } - - void dec_pending() { - CHECK(pending_ > 0); - --pending_; - if (pending_ == 0) { - promise_.set_result(std::move(result_)); - stop(); - } - } - - private: - td::Promise>> promise_; - size_t pending_ = 1; - std::vector> result_; - }; - auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); - - for (auto &[local_id, peer_pair] : peer_pairs_) { - td::actor::send_closure(callback, &Cb::inc_pending); - td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, - [local_id = local_id, peer_id = peer_id_short_, - callback](td::Result> R) { - if (R.is_error()) { - VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id - << " : " << R.move_as_error(); - td::actor::send_closure(callback, &Cb::dec_pending); - } else { - td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); - } - }); - } - td::actor::send_closure(callback, &Cb::dec_pending); -} - - void AdnlPeerPairImpl::got_data_from_db(td::Result R) { received_from_db_ = false; if (R.is_error()) { diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 1215f71da..4d6812a89 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -64,46 +64,10 @@ class AdnlPeerPair : public td::actor::Actor { static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); }; -class AdnlPeer : public td::actor::Actor { - public: - virtual void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket message, td::uint64 serialized_size) = 0; - virtual void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) = 0; - virtual void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) = 0; - void send_one_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - OutboundAdnlMessage message) { - std::vector vec; - vec.push_back(std::move(message)); - send_messages(src, src_mode, src_actor, std::move(vec)); - } - - void send_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - td::BufferSlice data, td::uint32 flags) { - auto M = OutboundAdnlMessage{adnlmessage::AdnlMessageCustom{std::move(data)}, flags}; - send_one_message(src, src_mode, src_actor, std::move(M)); - } - - static td::actor::ActorOwn create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id); - - virtual void del_local_id(AdnlNodeIdShort local_id) = 0; - virtual void update_id(AdnlNodeIdFull id) = 0; - virtual void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; - virtual void update_dht_node(td::actor::ActorId dht_node) = 0; - virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; - virtual void get_stats(bool all, td::Promise>> promise) = 0; -}; - } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 243974ba1..b1d983578 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -60,7 +60,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, + td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); void start_up() override; void alarm() override; @@ -209,7 +209,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::actor::ActorId network_manager_; td::actor::ActorId peer_table_; td::actor::ActorId local_actor_; - td::actor::ActorId peer_; td::actor::ActorId dht_node_; td::uint32 priority_ = 0; @@ -287,68 +286,12 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void prepare_packet_stats(); }; -class AdnlPeerImpl : public AdnlPeer { - public: - void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) override; - void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) override; - void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::string name, - td::Promise promise, td::Timestamp timeout, td::BufferSlice data, - td::uint32 flags) override; - - void del_local_id(AdnlNodeIdShort local_id) override; - void update_id(AdnlNodeIdFull id) override; - void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, - AdnlAddressList addr_list) override; - void update_dht_node(td::actor::ActorId dht_node) override; - void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; - void get_stats(bool all, td::Promise>> promise) override; - //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; - - AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) - : peer_id_short_(peer_id), dht_node_(dht_node), peer_table_(peer_table), network_manager_(network_manager) { - } - - struct PrintId { - AdnlNodeIdShort peer_id; - }; - - PrintId print_id() const { - return PrintId{peer_id_short_}; - } - - private: - AdnlNodeIdShort peer_id_short_; - AdnlNodeIdFull peer_id_; - std::map> peer_pairs_; - td::actor::ActorId dht_node_; - td::actor::ActorId peer_table_; - td::actor::ActorId network_manager_; -}; - } // namespace adnl } // namespace ton namespace td { -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl::PrintId &id) { - sb << "[peer " << id.peer_id << "]"; - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl &peer) { - sb << peer.print_id(); - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl *peer) { - sb << peer->print_id(); - return sb; -} - inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerPairImpl::PrintId &id) { sb << "[peerpair " << id.peer_id << "-" << id.local_id << "]"; return sb; diff --git a/adnl/adnl-static-nodes.cpp b/adnl/adnl-static-nodes.cpp deleted file mode 100644 index 251715be8..000000000 --- a/adnl/adnl-static-nodes.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl-static-nodes.h" -#include "adnl-static-nodes.hpp" - -#include "utils.hpp" - -namespace ton { - -namespace adnl { - -void AdnlStaticNodesManagerImpl::add_node(AdnlNode node) { - auto id_short = node.compute_short_id(); - VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; - - nodes_.emplace(id_short, std::move(node)); -} - -void AdnlStaticNodesManagerImpl::del_node(AdnlNodeIdShort id) { - nodes_.erase(id); -} - -td::Result AdnlStaticNodesManagerImpl::get_node(AdnlNodeIdShort id) { - auto it = nodes_.find(id); - if (it == nodes_.end()) { - return td::Status::Error(ErrorCode::notready, "static node not found"); - } - - return it->second; -} - -td::actor::ActorOwn AdnlStaticNodesManager::create() { - auto X = td::actor::create_actor("staticnodesmanager"); - return td::actor::ActorOwn(std::move(X)); -} - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-static-nodes.hpp b/adnl/adnl-static-nodes.hpp deleted file mode 100644 index 166be5aa1..000000000 --- a/adnl/adnl-static-nodes.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include -#include "adnl-static-nodes.h" - -namespace ton { - -namespace adnl { - -class AdnlStaticNodesManagerImpl : public AdnlStaticNodesManager { - public: - void add_node(AdnlNode node) override; - void del_node(AdnlNodeIdShort id) override; - td::Result get_node(AdnlNodeIdShort id) override; - AdnlStaticNodesManagerImpl() { - } - - private: - std::map nodes_; -}; - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-test-loopback-implementation.h b/adnl/adnl-test-loopback-implementation.h index 4773f0539..c2ff3207f 100644 --- a/adnl/adnl-test-loopback-implementation.h +++ b/adnl/adnl-test-loopback-implementation.h @@ -34,11 +34,16 @@ class TestLoopbackNetworkManager : public ton::adnl::AdnlNetworkManager { callback_ = std::move(callback); } + void install_tunnel_events_handler(std::unique_ptr handler) override {}; + void add_self_addr(td::IPAddress addr, AdnlCategoryMask cat_mask, td::uint32 priority) override { } void add_proxy_addr(td::IPAddress addr, td::uint16 local_port, std::shared_ptr proxy, AdnlCategoryMask cat_mask, td::uint32 priority) override { } + void add_tunnel(std::string global_config, std::string tunnel_config, AdnlCategoryMask cat_mask, td::uint32 priority, td::Promise on_ready, + td::actor::Scheduler* scheduler) override { + } void send_udp_packet(ton::adnl::AdnlNodeIdShort src_id, ton::adnl::AdnlNodeIdShort dst_id, td::IPAddress dst_addr, td::uint32 priority, td::BufferSlice data) override { if (allowed_sources_.count(src_id) == 0 || allowed_destinations_.count(dst_id) == 0) { diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 2a8cd0ec6..f6e269072 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -27,7 +27,7 @@ for file in ../artifacts/*; do printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop cp ../ton.png $appName.AppDir/$appName.png cp $file $appName.AppDir/usr/bin/ - cp ../build/openssl_3/libcrypto.so.3 \ + cp ../openssl_3/libcrypto.so.3 \ /lib/$ARCH-linux-gnu/libatomic.so.1 \ /lib/$ARCH-linux-gnu/libsodium.so.23 \ /lib/$ARCH-linux-gnu/libz.so.1 \ @@ -35,6 +35,9 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libmicrohttpd.so.12 \ /lib/$ARCH-linux-gnu/libreadline.so.8 \ /lib/$ARCH-linux-gnu/libstdc++.so.6 \ + /lib/$ARCH-linux-gnu/libgsl.so.27 \ + /lib/$ARCH-linux-gnu/libblas.so.3 \ + /lib/$ARCH-linux-gnu/libgslcblas.so.0 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index b785339e8..302effb71 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja pkg-config automake libtool autoconf texinfo +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang @@ -35,77 +48,79 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j12 +make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. +cd ../../build # ./lib/liblz4.a # ./lib else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium-1.0.18" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + cd ../3pp + curl -LO https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz + tar -xzf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j4 test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium-1.0.18 echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j4 test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j4 test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -113,6 +128,7 @@ cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ @@ -136,16 +152,16 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -170,6 +186,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ @@ -181,9 +199,3 @@ if [ "$with_artifacts" = true ]; then rsync -r crypto/fift/lib artifacts/ chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure -fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 5c4a5fe07..c2aa09ced 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja libsodium libmicrohttpd pkg-config automake libtool autoconf gnutls +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ @@ -34,14 +47,13 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then git clone https://github.com/lz4/lz4 cd lz4 lz4Path=`pwd` git checkout v1.9.4 - make -j12 + make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else @@ -49,12 +61,26 @@ else echo "Using compiled lz4" fi +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j4 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + brew unlink openssl@1.1 brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib @@ -64,16 +90,16 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -98,6 +124,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ @@ -110,8 +138,3 @@ if [ "$with_artifacts" = true ]; then chmod -R +x artifacts/* fi -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 4e63234d9..2cdc6c652 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -2,17 +2,27 @@ with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -23,19 +33,18 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -51,18 +60,18 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + validator-engine lite-client validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -95,6 +104,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts @@ -104,6 +114,5 @@ fi if [ "$with_tests" = true ]; then cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" ctest --output-on-failure --timeout 1800 fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index 2f0a1ba4d..c7bffd40e 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -1,18 +1,29 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev ccache with_artifacts=false +with_ccache=false -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -21,79 +32,80 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) -export CCACHE_DISABLE=1 +export CC=$(which clang-16) +export CXX=$(which clang++-16) -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + mkdir -p ../3pp/libsodium + wget -O ../3pp/libsodium/libsodium-1.0.18.tar.gz https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz + cd ../3pp/libsodium + tar xf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium/libsodium-1.0.18 echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -117,11 +129,10 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } ninja tonlibjson emulator -test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +test $? -eq 0 || { echo "Can't compile tonlibjson and emulator"; exit 1; } cd .. diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 16e77ac8d..a4f0bb233 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -6,16 +6,26 @@ with_tests=false with_artifacts=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 + rm -rf ~/.ccache +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -24,79 +34,79 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) -export CCACHE_DISABLE=1 +export CC=$(which clang-16) +export CXX=$(which clang++-16) -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium + cd ../3pp/libsodium sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -120,23 +130,22 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + validator-engine lite-client validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -158,6 +167,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 49cc8e1ea..6ddbaf885 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,21 +1,31 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -26,19 +36,19 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 + +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -53,18 +63,18 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + validator-engine lite-client validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -78,6 +88,7 @@ ldd ./validator-engine/validator-engine || exit 1 cd .. + if [ "$with_artifacts" = true ]; then rm -rf artifacts mkdir artifacts @@ -88,15 +99,10 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts cp -R crypto/fift/lib artifacts chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 844c09fcd..e671d277c 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -136,18 +145,18 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip @@ -193,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat new file mode 100644 index 000000000..6ff7166af --- /dev/null +++ b/assembly/native/build-windows-2022.bat @@ -0,0 +1,207 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM +IF %errorlevel% NEQ 0 ( + echo Can't install nasm + exit /b %errorlevel% +) + +if not exist "third_libs" ( + mkdir "third_libs" +) +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using zlib... +) + +if not exist "lz4" ( + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2022\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using lz4... +) + +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd .. +) else ( + echo Using libsodium... +) + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... +) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2022 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v143 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + +cd .. +echo Current dir %cd% + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2022\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2022\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +echo Strip and copy artifacts +cd .. +echo where strip +where strip +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows-github-2022.bat b/assembly/native/build-windows-github-2022.bat new file mode 100644 index 000000000..f1e4b884a --- /dev/null +++ b/assembly/native/build-windows-github-2022.bat @@ -0,0 +1,2 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows-2022.bat -t diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 68f83c394..f0a65670b 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -136,18 +145,18 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip @@ -193,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index a463c02aa..a80430c1d 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -2,7 +2,7 @@ # sudo apt update # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \ # libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip \ -# nodejs libsodium-dev automake libtool libjemalloc-dev +# nodejs libsodium-dev automake libtool libjemalloc-dev ccache # wget https://apt.llvm.org/llvm.sh # chmod +x llvm.sh @@ -30,19 +30,18 @@ if [ "$scratch_new" = true ]; then rm -rf openssl zlib lz4 emsdk libsodium build openssl_em fi - -if [ ! -d "openssl" ]; then - git clone https://github.com/openssl/openssl.git - cp -r openssl openssl_em - cd openssl +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make -j16 - OPENSSL_DIR=`pwd` + make build_libs -j$(nproc) + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } cd .. else - OPENSSL_DIR=`pwd`/openssl - echo Using compiled openssl at $OPENSSL_DIR + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" fi if [ ! -d "build" ]; then @@ -50,9 +49,9 @@ if [ ! -d "build" ]; then cd build cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ - -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \ - -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ - -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so + -DOPENSSL_ROOT_DIR=$opensslPath \ + -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ + -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so test $? -eq 0 || { echo "Can't configure TON build"; exit 1; } ninja fift smc-envelope @@ -82,61 +81,63 @@ export CCACHE_DISABLE=1 cd .. -if [ ! -f "openssl_em/openssl_em" ]; then - cd openssl_em +if [ ! -f "3pp_emscripten/openssl_em/openssl_em" ]; then + mkdir -p 3pp_emscripten + git clone https://github.com/openssl/openssl 3pp_emscripten/openssl_em + cd 3pp_emscripten/openssl_em emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile sed -i 's/-O3/-Os/g' Makefile emmake make depend - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; } - OPENSSL_DIR=`pwd` + opensslPath=`pwd` touch openssl_em - cd .. + cd ../.. else - OPENSSL_DIR=`pwd`/openssl_em - echo Using compiled with empscripten openssl at $OPENSSL_DIR + opensslPath=`pwd`/3pp_emscripten/openssl_em + echo Using compiled with empscripten openssl at $opensslPath fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "3pp_emscripten/zlib" ]; then + git clone https://github.com/madler/zlib.git 3pp_emscripten/zlib + cd 3pp_emscripten/zlib git checkout v1.3.1 ZLIB_DIR=`pwd` emconfigure ./configure --static - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; } - cd .. + cd ../.. else - ZLIB_DIR=`pwd`/zlib + ZLIB_DIR=`pwd`/3pp_emscripten/zlib echo Using compiled zlib with emscripten at $ZLIB_DIR fi -if [ ! -d "lz4" ]; then - git clone https://github.com/lz4/lz4.git - cd lz4 +if [ ! -d "3pp_emscripten/lz4" ]; then + git clone https://github.com/lz4/lz4.git 3pp_emscripten/lz4 + cd 3pp_emscripten/lz4 git checkout v1.9.4 LZ4_DIR=`pwd` - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; } - cd .. + cd ../.. else - LZ4_DIR=`pwd`/lz4 + LZ4_DIR=`pwd`/3pp_emscripten/lz4 echo Using compiled lz4 with emscripten at $LZ4_DIR fi -if [ ! -d "libsodium" ]; then - git clone https://github.com/jedisct1/libsodium - cd libsodium +if [ ! -d "3pp_emscripten/libsodium" ]; then + git clone https://github.com/jedisct1/libsodium 3pp_emscripten/libsodium + cd 3pp_emscripten/libsodium git checkout 1.0.18-RELEASE SODIUM_DIR=`pwd` emconfigure ./configure --disable-ssp - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; } - cd .. + cd ../.. else - SODIUM_DIR=`pwd`/libsodium + SODIUM_DIR=`pwd`/3pp_emscripten/libsodium echo Using compiled libsodium with emscripten at $SODIUM_DIR fi @@ -150,8 +151,8 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \ -DOPENSSL_FOUND=1 \ --DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ --DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \ -DSODIUM_FOUND=1 \ @@ -163,7 +164,7 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; } cp -R ../crypto/smartcont ../crypto/fift/lib crypto -emmake make -j16 funcfiftlib func fift tlbc emulator-emscripten +emmake make -j$(nproc) funcfiftlib func fift tlbc emulator-emscripten test $? -eq 0 || { echo "Can't compile TON with emmake "; exit 1; } diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 86cb3e740..afd304177 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(NIX "Use \"ON\" for a static build." OFF) set(BLOCHAIN_EXPLORER_SOURCE diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 26a6787e1..6919c5c14 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -525,8 +525,8 @@ HttpQueryBlockSearch::HttpQueryBlockSearch(std::map op } if (opts.count("utime") == 1) { try { - seqno_ = static_cast(std::stoull(opts["utime"])); - mode_ = 1; + utime_ = static_cast(std::stoull(opts["utime"])); + mode_ = 4; } catch (...) { error_ = td::Status::Error("cannot parse utime"); return; @@ -1429,10 +1429,10 @@ void HttpQueryStatus::finish_query() { A << "" << static_cast(x->ts_.at_unix()) << ""; } A << "\n"; - for (td::uint32 i = 0; i < results_.ips.size(); i++) { + for (td::uint32 i = 0; i < results_.addrs.size(); i++) { A << ""; - if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; + if (!results_.addrs[i].empty()) { + A << "" << results_.addrs[i] << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index ca50d5266..d21bc465c 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -187,7 +187,7 @@ class CoreActor : public CoreActorInterface { std::mutex queue_mutex_; std::mutex res_mutex_; std::map> results_; - std::vector addrs_; + std::vector addrs_; static CoreActor* instance_; td::actor::ActorId self_id_; @@ -220,7 +220,7 @@ class CoreActor : public CoreActorInterface { } void get_results(td::uint32 max, td::Promise promise) override { RemoteNodeStatusList r; - r.ips = hide_ips_ ? std::vector{addrs_.size()} : addrs_; + r.addrs = hide_ips_ ? std::vector{addrs_.size()} : addrs_; auto it = results_.rbegin(); while (it != results_.rend() && r.results.size() < max) { r.results.push_back(it->second); @@ -445,14 +445,14 @@ class CoreActor : public CoreActorInterface { r_servers.ensure(); servers = r_servers.move_as_ok(); for (const auto& serv : servers) { - addrs_.push_back(serv.addr); + addrs_.push_back(serv.hostname); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); + addrs_.push_back(servers.back().hostname); } n_servers_ = servers.size(); client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362de..abbbde9e9 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -59,7 +59,7 @@ class CoreActorInterface : public td::actor::Actor { }; struct RemoteNodeStatusList { - std::vector ips; + std::vector addrs; std::vector> results; }; virtual ~CoreActorInterface() = default; diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index 3f766688e..6175cb86b 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index a6160383c..b663cfc06 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -528,6 +528,7 @@ void CatChainReceiverImpl::start_up() { } overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 88a3671b3..9252178b5 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(COMMON_SOURCE checksum.h errorcode.h diff --git a/common/global-version.h b/common/global-version.h index 2308ce3e9..d88a22c32 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 10; +constexpr int SUPPORTED_VERSION = 12; } diff --git a/create-hardfork/CMakeLists.txt b/create-hardfork/CMakeLists.txt index 3024603c2..daa7f2e73 100644 --- a/create-hardfork/CMakeLists.txt +++ b/create-hardfork/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 72bffae56..aadf877ca 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -216,8 +216,9 @@ class HardforkCreator : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -246,7 +247,7 @@ class HardforkCreator : public td::actor::Actor { void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } @@ -257,8 +258,8 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -279,9 +280,6 @@ class HardforkCreator : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381..50f890394 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -12,14 +10,11 @@ set(TON_CRYPTO_CORE_SOURCE common/bigexp.cpp common/bitstring.cpp common/util.cpp - ellcurve/Ed25519.cpp - ellcurve/Fp25519.cpp - ellcurve/Montgomery.cpp - ellcurve/TwEdwards.cpp openssl/bignum.cpp openssl/residue.cpp openssl/rand.cpp vm/boc.cpp + vm/boc-compression.cpp vm/large-boc-serializer.cpp tl/tlblib.cpp @@ -34,11 +29,6 @@ set(TON_CRYPTO_CORE_SOURCE common/linalloc.hpp common/promiseop.hpp - ellcurve/Ed25519.h - ellcurve/Fp25519.h - ellcurve/Montgomery.h - ellcurve/TwEdwards.h - openssl/bignum.h openssl/digest.hpp openssl/rand.hpp @@ -72,7 +62,6 @@ set(TON_CRYPTO_CORE_SOURCE vm/cells/CellString.h vm/cells/CellTraits.h vm/cells/CellUsageTree.h - vm/cells/CellWithStorage.h vm/cells/DataCell.h vm/cells/ExtCell.h vm/cells/LevelMask.h @@ -120,6 +109,7 @@ set(TON_CRYPTO_SOURCE vm/arithops.h vm/atom.h vm/boc.h + vm/boc-compression.h vm/boc-writers.h vm/box.hpp vm/cellops.h @@ -144,6 +134,7 @@ set(TON_CRYPTO_SOURCE set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp + vm/db/DynamicBagOfCellsDbV2.cpp vm/db/CellStorage.cpp vm/db/TonDb.cpp @@ -214,6 +205,8 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/account-storage-stat.h + block/account-storage-stat.cpp block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} @@ -352,10 +345,6 @@ target_include_directories(ton_db PUBLIC $) target_link_libraries(ton_db PUBLIC tdutils tddb ton_crypto) -add_executable(test-ed25519-crypto test/test-ed25519-crypto.cpp) -target_include_directories(test-ed25519-crypto PUBLIC $) -target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto) - add_library(fift-lib STATIC ${FIFT_SOURCE}) target_include_directories(fift-lib PUBLIC $) target_link_libraries(fift-lib PUBLIC ton_crypto) @@ -421,16 +410,8 @@ if (WINGETOPT_FOUND) target_link_libraries_system(tlbc wingetopt) endif() -add_library(pow-miner-lib util/Miner.cpp util/Miner.h) -target_include_directories(pow-miner-lib PUBLIC $) -target_link_libraries(pow-miner-lib PUBLIC ton_crypto) - -add_executable(pow-miner util/pow-miner.cpp) -target_link_libraries(pow-miner PRIVATE ton_crypto pow-miner-lib git) - if (WINGETOPT_FOUND) target_link_libraries_system(fift wingetopt) - target_link_libraries_system(pow-miner wingetopt) endif() add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) @@ -541,7 +522,7 @@ target_include_directories(create-state PUBLIC $ -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - #include "td/utils/base64.h" #include "td/utils/BigNum.h" #include "td/utils/format.h" @@ -35,12 +31,6 @@ #include #include -#else - -#include "crypto/ellcurve/Ed25519.h" - -#endif - namespace td { Ed25519::PublicKey::PublicKey(SecureString octet_string) : octet_string_(std::move(octet_string)) { @@ -57,8 +47,6 @@ SecureString Ed25519::PrivateKey::as_octet_string() const { return octet_string_.copy(); } -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - namespace detail { static Result X25519_key_from_PKEY(EVP_PKEY *pkey, bool is_private) { @@ -249,7 +237,7 @@ Result Ed25519::compute_shared_secret(const PublicKey &public_key, BigNum::mod_sub(y2, p, y2, p, context); BigNum inverse_y_plus_1; - BigNum::mod_inverse(inverse_y_plus_1, y2, p, context); + TRY_STATUS(BigNum::mod_inverse(inverse_y_plus_1, y2, p, context)); BigNum u; BigNum::mod_mul(u, y, inverse_y_plus_1, p, context); @@ -314,89 +302,4 @@ int Ed25519::version() { return OPENSSL_VERSION_NUMBER; } -#else - -Result Ed25519::generate_private_key() { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.random_private_key(true)) { - return Status::Error("Can't generate random private key"); - } - SecureString private_key_buf(32); - if (!private_key.export_private_key(private_key_buf.as_mutable_slice())) { - return Status::Error("Failed to export private key"); - } - return PrivateKey(std::move(private_key_buf)); -} - -Result Ed25519::PrivateKey::get_public_key() const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString public_key(32); - if (!private_key.get_public_key().export_public_key(public_key.as_mutable_slice())) { - return Status::Error("Failed to export public key"); - } - return PublicKey(std::move(public_key)); -} - -Result Ed25519::PrivateKey::as_pem(Slice password) const { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::from_pem(Slice pem, Slice password) { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::sign(Slice data) const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString signature(crypto::Ed25519::sign_bytes, '\0'); - if (!private_key.sign_message(signature.as_mutable_slice(), data)) { - return Status::Error("Failed to sign message"); - } - return std::move(signature); -} - -Status Ed25519::PublicKey::verify_signature(Slice data, Slice signature) const { - if (signature.size() != crypto::Ed25519::sign_bytes) { - return Status::Error("Signature has invalid length"); - } - - crypto::Ed25519::PublicKey public_key; - if (!public_key.import_public_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad public key"); - } - if (public_key.check_message_signature(signature, data)) { - return Status::OK(); - } - return Status::Error("Wrong signature"); -} - -Result Ed25519::compute_shared_secret(const PublicKey &public_key, const PrivateKey &private_key) { - crypto::Ed25519::PrivateKey tmp_private_key; - if (!tmp_private_key.import_private_key(Slice(private_key.as_octet_string()).ubegin())) { - return Status::Error("Bad private key"); - } - crypto::Ed25519::PublicKey tmp_public_key; - if (!tmp_public_key.import_public_key(Slice(public_key.as_octet_string()).ubegin())) { - return Status::Error("Bad public key"); - } - SecureString shared_secret(32, '\0'); - if (!tmp_private_key.compute_shared_secret(shared_secret.as_mutable_slice(), tmp_public_key)) { - return Status::Error("Failed to compute shared secret"); - } - return std::move(shared_secret); -} - -int Ed25519::version() { - return 0; -} - -#endif - } // namespace td - -#endif diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp new file mode 100644 index 000000000..a6159c224 --- /dev/null +++ b/crypto/block/account-storage-stat.cpp @@ -0,0 +1,282 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "account-storage-stat.h" + +namespace block { + +AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { +} + +AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, + td::uint64 total_cells, td::uint64 total_bits) + : dict_(std::move(dict_root), 256), total_cells_(total_cells), total_bits_(total_bits), roots_(std::move(roots)) { +} + +AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) + : dict_(parent->dict_) + , dict_up_to_date_(parent->dict_up_to_date_) + , total_cells_(parent->total_cells_) + , total_bits_(parent->total_bits_) + , roots_(parent->roots_) + , parent_(parent) { + CHECK(parent_->parent_ == nullptr); +} + +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + dict_up_to_date_ = true; + parent_ = nullptr; + return td::Status::OK(); + } + + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + if (to_add.empty() && to_del.empty()) { + return td::Status::OK(); + } + + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); + if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { + return td::Status::Error(errorcode_limits_exceeded, "too big Merkle depth"); + } + } + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); + } + + roots_ = std::move(new_roots); + dict_up_to_date_ = false; + for (auto& [_, e] : cache_) { + TRY_STATUS(finalize_entry(e)); + } + return td::Status::OK(); +} + +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_from_dict(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + e.max_merkle_depth = 0; + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + +td::Result AccountStorageStat::add_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known) { + TRY_STATUS(fetch_from_dict(e)); + } + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_from_dict(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; + } + + td::uint32 max_merkle_depth = 0; + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof || + cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + ++max_merkle_depth; + } + max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); + Entry& e2 = get_entry(cell); + e2.max_merkle_depth = max_merkle_depth; + ++total_cells_; + total_bits_ += cs.size(); + return CellInfo{max_merkle_depth}; +} + +td::Status AccountStorageStat::remove_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known) { + TRY_STATUS(fetch_from_dict(e)); + } + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); + } + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_from_dict(e)); + } + if (e.refcnt_diff >= 0 || e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); + } + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(remove_cell(cs.prefetch_ref(i))); + } + --total_cells_; + total_bits_ -= cs.size(); + return td::Status::OK(); +} + +td::Result> AccountStorageStat::get_dict_root() { + if (!dict_up_to_date_) { + std::vector>> values; + if (parent_ && !parent_->dict_up_to_date_) { + for (auto& [_, ep] : parent_->cache_) { + if (ep.dict_refcnt_diff == 0) { + continue; + } + Entry& e = cache_[ep.hash]; + if (!e.inited) { + e = ep; + } + } + } + for (auto& [_, e] : cache_) { + if (e.dict_refcnt_diff == 0) { + continue; + } + if (!e.exists_known || !e.refcnt || (e.exists && !e.max_merkle_depth)) { + return td::Status::Error("unexpected state of storage stat"); + } + if (e.exists) { + Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); + } else { + values.emplace_back(e.hash.bits(), Ref{}); + } + e.dict_refcnt_diff = 0; + } + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); + } + dict_up_to_date_ = true; + } + return dict_.get_root_cell(); +} + +void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { + CHECK(parent_ == nullptr); + if (child.parent_ == nullptr) { + *this = std::move(child); + return; + } + CHECK(child.parent_ == this); + total_bits_ = child.total_bits_; + total_cells_ = child.total_cells_; + dict_ = std::move(child.dict_); + dict_up_to_date_ = child.dict_up_to_date_; + roots_ = std::move(child.roots_); + for (auto& [hash, e] : child.cache_) { + cache_[hash] = std::move(e); + } +} + +AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { + Entry& e = cache_[cell->get_hash()]; + if (e.inited) { + return e; + } + if (parent_) { + auto it = parent_->cache_.find(cell->get_hash()); + if (it != parent_->cache_.end()) { + CHECK(it->second.inited); + e = it->second; + return e; + } + } + e.inited = true; + e.hash = cell->get_hash(); + return e; +} + +td::Status AccountStorageStat::fetch_from_dict(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.hash.as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; + } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.hash.to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.hash.to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::finalize_entry(Entry& e) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_from_dict(e)); + e.refcnt.value() += e.refcnt_diff; + e.dict_refcnt_diff += e.refcnt_diff; + e.refcnt_diff = 0; + e.exists = (e.refcnt.value() != 0); + if (e.exists && !e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Error on entry " << e.hash.to_hex() << " : unknown merkle depth"); + } + return td::Status::OK(); +} + +} // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h new file mode 100644 index 000000000..4ada076b8 --- /dev/null +++ b/crypto/block/account-storage-stat.h @@ -0,0 +1,114 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/dict.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" +#include "common/bitstring.h" +#include "block.h" +#include "vm/db/CellHashTable.h" + +namespace block { +using td::Ref; + +class AccountStorageStat { + public: + AccountStorageStat(); + AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, + td::uint64 total_bits); + explicit AccountStorageStat(const AccountStorageStat *parent); + AccountStorageStat(const AccountStorageStat &other) = delete; + AccountStorageStat(AccountStorageStat &&other) = default; + ~AccountStorageStat() = default; + + AccountStorageStat &operator=(const AccountStorageStat &other) = delete; + AccountStorageStat &operator=(AccountStorageStat &&other) = default; + + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + void add_hint(const td::HashSet &visited); + + td::uint64 get_total_cells() const { + return total_cells_; + } + + td::uint64 get_total_bits() const { + return total_bits_; + } + + td::Result> get_dict_root(); + + td::Result get_dict_hash() { + TRY_RESULT(root, get_dict_root()); + return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; + } + + bool is_dict_ready() const { + return dict_up_to_date_; + } + + void apply_child_stat(AccountStorageStat &&child); + + static constexpr int errorcode_limits_exceeded = 999; + + private: + vm::Dictionary dict_; + bool dict_up_to_date_ = true; + td::uint64 total_cells_, total_bits_; + std::vector> roots_; + const AccountStorageStat *parent_ = nullptr; + + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + + td::Result add_cell(const Ref &cell); + td::Status remove_cell(const Ref &cell); + + struct Entry { + bool inited = false; + vm::CellHash hash; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; + td::int32 dict_refcnt_diff = 0; + }; + + td::HashMap> cache_; + + Entry &get_entry(const Ref &cell); + td::Status fetch_from_dict(Entry &e); + td::Status finalize_entry(Entry &e); + + static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; + static constexpr td::uint32 MAX_MERKLE_DEPTH = 2; +}; + +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + +} // namespace block diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 50851c795..9443b69f3 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -665,7 +665,7 @@ bool CommonMsgInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && t_MsgAddressInt.validate_skip(ops, cs, weak) // src && t_MsgAddressInt.validate_skip(ops, cs, weak) // dest && t_CurrencyCollection.validate_skip(ops, cs, weak) // value - && t_Grams.validate_skip(ops, cs, weak) // ihr_fee + && t_Grams.validate_skip(ops, cs, weak) // extra_flags && t_Grams.validate_skip(ops, cs, weak) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -684,7 +684,7 @@ bool CommonMsgInfo::unpack(vm::CellSlice& cs, CommonMsgInfo::Record_int_msg_info return get_tag(cs) == int_msg_info && cs.advance(1) && cs.fetch_bool_to(data.ihr_disabled) && cs.fetch_bool_to(data.bounce) && cs.fetch_bool_to(data.bounced) && t_MsgAddressInt.fetch_to(cs, data.src) && t_MsgAddressInt.fetch_to(cs, data.dest) && t_CurrencyCollection.fetch_to(cs, data.value) && - t_Grams.fetch_to(cs, data.ihr_fee) && t_Grams.fetch_to(cs, data.fwd_fee) && + t_Grams.fetch_to(cs, data.extra_flags) && t_Grams.fetch_to(cs, data.fwd_fee) && cs.fetch_uint_to(64, data.created_lt) && cs.fetch_uint_to(32, data.created_at); } @@ -696,7 +696,7 @@ bool CommonMsgInfo::skip(vm::CellSlice& cs) const { && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -718,7 +718,7 @@ bool CommonMsgInfo::get_created_lt(vm::CellSlice& cs, unsigned long long& create && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.fetch_ulong_bool(64, created_lt) // created_lt:uint64 && cs.advance(32); // created_at:uint32 @@ -738,7 +738,7 @@ const TickTock t_TickTock; const RefAnything t_RefCell; bool StateInit::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return Maybe{5}.validate_skip(ops, cs, weak) // split_depth:(Maybe (## 5)) + return Maybe{5}.validate_skip(ops, cs, weak) // fixed_prefix_length:(Maybe (## 5)) && Maybe{}.validate_skip(ops, cs, weak) // special:(Maybe TickTock) && Maybe{}.validate_skip(ops, cs, weak) // code:(Maybe ^Cell) && Maybe{}.validate_skip(ops, cs, weak) // data:(Maybe ^Cell) @@ -960,41 +960,34 @@ const MsgEnvelope t_MsgEnvelope; const RefTo t_Ref_MsgEnvelope; bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7) -} - -bool StorageUsed::skip(vm::CellSlice& cs) const { - return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) - && t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7) - && t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7) -} - -const StorageUsed t_StorageUsed; - -bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) && t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7) } -bool StorageUsedShort::skip(vm::CellSlice& cs) const { +bool StorageUsed::skip(vm::CellSlice& cs) const { return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) && t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7) } -const StorageUsedShort t_StorageUsedShort; +const StorageUsed t_StorageUsed; const Maybe t_Maybe_Grams; bool StorageInfo::skip(vm::CellSlice& cs) const { - return t_StorageUsed.skip(cs) // used:StorageUsed - && cs.advance(32) // last_paid:uint32 - && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) + int extra_tag = 0; + return t_StorageUsed.skip(cs) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + && cs.advance(32) // last_paid:uint32 + && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) } bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + int extra_tag = 0; + return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || + (extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; && cs.advance(32) // last_paid:uint32 && t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams) } @@ -1085,7 +1078,7 @@ bool Account::skip_copy_depth_balance(vm::CellBuilder& cb, vm::CellSlice& cs) co case account: return cs.advance(1) // account$1 && t_MsgAddressInt.skip_get_depth(cs, depth) // addr:MsgAddressInt - && cb.store_uint_leq(30, depth) // -> store split_depth:(#<= 30) + && cb.store_uint_leq(30, depth) // -> store fixed_prefix_length:(#<= 30) && t_StorageInfo.skip(cs) // storage_stat:StorageInfo && t_AccountStorage.skip_copy_balance(cb, cs); // storage:AccountStorage } @@ -1230,13 +1223,14 @@ bool HashmapAugE::extract_extra(vm::CellSlice& cs) const { bool DepthBalanceInfo::skip(vm::CellSlice& cs) const { return cs.advance(5) && t_CurrencyCollection.skip( - cs); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; + cs); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; } bool DepthBalanceInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.fetch_ulong(5) <= 30 && - t_CurrencyCollection.validate_skip(ops, cs, - weak); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection + t_CurrencyCollection.validate_skip( + ops, cs, + weak); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection } bool DepthBalanceInfo::null_value(vm::CellBuilder& cb) const { @@ -1368,7 +1362,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const { && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { @@ -1381,7 +1375,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1392,11 +1386,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs) // msg_fees:Grams && t_Grams.skip(cs); // fwd_fees:Grams } @@ -1409,11 +1403,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } @@ -1849,26 +1843,18 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { +static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags)); +} + +bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { int tag = get_tag(cs); switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination - if (cs.advance(3) && cs.size_refs() >= 3) { - auto msg_cs = load_cell_slice(cs.fetch_ref()); - CommonMsgInfo::Record_int_msg_info msg_info; - td::RefInt256 ihr_fee; - vm::CellBuilder aux; - // sort of Prolog-style in C++ - return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && - cs.fetch_ref().not_null() && (ihr_fee = t_Grams.as_integer_skip(cs)).not_null() && - cs.fetch_ref().not_null() && !cmp(ihr_fee, t_Grams.as_integer(*msg_info.ihr_fee)) && - cb.append_cellslice_bool(msg_info.ihr_fee) // fees_collected := ihr_fee - && aux.append_cellslice_bool(msg_info.ihr_fee) && t_ExtraCurrencyCollection.null_value(aux) && - t_CurrencyCollection.add_values(cb, aux.as_cellslice_ref().write(), - msg_info.value.write()); // value_imported := ihr_fee + value - } + // IHR is not implemented return false; case msg_import_imm: // internal message re-imported from this very block if (cs.advance(3) && cs.size_refs() >= 2) { @@ -1894,7 +1880,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && cb.append_cellslice_bool(in_msg.fwd_fee_remaining) // fees_collected := fwd_fee_remaining && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1917,7 +1903,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && t_Grams.store_integer_ref(cb, std::move(transit_fee)) // fees_collected := transit_fees && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1947,8 +1933,8 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { const InMsg t_InMsg; -const Aug_InMsgDescr aug_InMsgDescr; -const InMsgDescr t_InMsgDescr; +const Aug_InMsgDescr aug_InMsgDescrDefault(ton::SUPPORTED_VERSION); +const InMsgDescr t_InMsgDescrDefault(ton::SUPPORTED_VERSION); bool OutMsg::skip(vm::CellSlice& cs) const { switch (get_tag(cs)) { @@ -2044,7 +2030,7 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { +bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { auto tag = get_tag(cs); switch (tag) { case msg_export_ext: // external outbound message carries no value @@ -2077,7 +2063,7 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { td::RefInt256 value_grams, ihr_fee, fwd_fee_remaining; return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && (value_grams = t_Grams.as_integer_skip(msg_info.value.write())).not_null() && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && (fwd_fee_remaining = t_Grams.as_integer(out_msg.fwd_fee_remaining)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool(std::move(msg_info.value)); @@ -2118,8 +2104,8 @@ bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) c const OutMsg t_OutMsg; -const Aug_OutMsgDescr aug_OutMsgDescr; -const OutMsgDescr t_OutMsgDescr; +const Aug_OutMsgDescr aug_OutMsgDescrDefault(ton::SUPPORTED_VERSION); +const OutMsgDescr t_OutMsgDescrDefault(ton::SUPPORTED_VERSION); bool EnqueuedMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(64) && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 65f8b91fe..2eb8c5d77 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -405,7 +405,7 @@ struct CommonMsgInfo final : TLB_Complex { struct CommonMsgInfo::Record_int_msg_info { bool ihr_disabled, bounce, bounced; - Ref src, dest, value, ihr_fee, fwd_fee; + Ref src, dest, value, extra_flags, fwd_fee; unsigned long long created_lt; unsigned created_at; }; @@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex { extern const StorageUsed t_StorageUsed; -struct StorageUsedShort final : TLB_Complex { - bool skip(vm::CellSlice& cs) const override; - bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; -}; - -extern const StorageUsedShort t_StorageUsedShort; - struct StorageInfo final : TLB_Complex { bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; @@ -611,7 +604,7 @@ extern const Aug_ShardAccounts aug_ShardAccounts; struct ShardAccounts final : TLB_Complex { HashmapAugE dict_type; - ShardAccounts() : dict_type(256, aug_ShardAccounts){}; + ShardAccounts() : dict_type(256, aug_ShardAccounts) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -821,7 +814,7 @@ struct InMsg final : TLB_Complex { } return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } - bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; }; extern const InMsg t_InMsg; @@ -851,7 +844,7 @@ struct OutMsg final : TLB_Complex { } return t; } - bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; @@ -860,18 +853,22 @@ extern const OutMsg t_OutMsg; // next: InMsgDescr, OutMsgDescr, OutMsgQueue, and their augmentations struct Aug_InMsgDescr final : AugmentationCheckData { - Aug_InMsgDescr() : AugmentationCheckData(t_InMsg, t_ImportFees) { + explicit Aug_InMsgDescr(int global_version) + : AugmentationCheckData(t_InMsg, t_ImportFees), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_InMsg.get_import_fees(cb, cs); + return t_InMsg.get_import_fees(cb, cs, global_version); } + int global_version; }; -extern const Aug_InMsgDescr aug_InMsgDescr; +extern const Aug_InMsgDescr aug_InMsgDescrDefault; struct InMsgDescr final : TLB_Complex { + Aug_InMsgDescr aug; HashmapAugE dict_type; - InMsgDescr() : dict_type(256, aug_InMsgDescr){}; + explicit InMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -880,21 +877,25 @@ struct InMsgDescr final : TLB_Complex { } }; -extern const InMsgDescr t_InMsgDescr; +extern const InMsgDescr t_InMsgDescrDefault; struct Aug_OutMsgDescr final : AugmentationCheckData { - Aug_OutMsgDescr() : AugmentationCheckData(t_OutMsg, t_CurrencyCollection) { + explicit Aug_OutMsgDescr(int global_version) + : AugmentationCheckData(t_OutMsg, t_CurrencyCollection), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_OutMsg.get_export_value(cb, cs); + return t_OutMsg.get_export_value(cb, cs, global_version); } + int global_version; }; -extern const Aug_OutMsgDescr aug_OutMsgDescr; +extern const Aug_OutMsgDescr aug_OutMsgDescrDefault; struct OutMsgDescr final : TLB_Complex { + Aug_OutMsgDescr aug; HashmapAugE dict_type; - OutMsgDescr() : dict_type(256, aug_OutMsgDescr){}; + explicit OutMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -903,7 +904,7 @@ struct OutMsgDescr final : TLB_Complex { } }; -extern const OutMsgDescr t_OutMsgDescr; +extern const OutMsgDescr t_OutMsgDescrDefault; struct EnqueuedMsg final : TLB_Complex { int get_size(const vm::CellSlice& cs) const override { @@ -942,7 +943,7 @@ extern const Aug_DispatchQueue aug_DispatchQueue; struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; - OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; + OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -1145,8 +1146,8 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; // Validate dict of libraries in message: used when sending and receiving message -bool validate_message_libs(const td::Ref &cell); -bool validate_message_relaxed_libs(const td::Ref &cell); +bool validate_message_libs(const td::Ref& cell); +bool validate_message_relaxed_libs(const td::Ref& cell); } // namespace tlb } // namespace block diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index e07822403..0893d630f 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -676,10 +676,25 @@ bool ParamLimits::deserialize(vm::CellSlice& cs) { } bool BlockLimits::deserialize(vm::CellSlice& cs) { - return cs.fetch_ulong(8) == 0x5d // block_limits#5d - && bytes.deserialize(cs) // bytes:ParamLimits - && gas.deserialize(cs) // gas:ParamLimits - && lt_delta.deserialize(cs); // lt_delta:ParamLimits + auto tag = cs.fetch_ulong(8); + if (tag != 0x5d && tag != 0x5e) { + return false; + } + // block_limits#5d + // block_limits_v2#5e + bool ok = bytes.deserialize(cs) // bytes:ParamLimits + && gas.deserialize(cs) // gas:ParamLimits + && lt_delta.deserialize(cs); // lt_delta:ParamLimits + if (!ok) { + return false; + } + if (tag == 0x5e) { + return collated_data.deserialize(cs) && // collated_data:ParamLimits + imported_msg_queue.deserialize(cs); // imported_msg_queue:ImportedMsgQueueLimits + } else { + collated_data = bytes; + return true; + } } int ParamLimits::classify(td::uint64 value) const { @@ -711,12 +726,19 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const { return lt_delta.classify(lt - start_lt); } -int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const { - return std::max(std::max(classify_size(size), classify_gas(gas)), classify_lt(lt)); +int BlockLimits::classify_collated_data_size(td::uint64 size) const { + return collated_data.classify(size); +} + +int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const { + return std::max( + {classify_size(size), classify_gas(gas), classify_lt(lt), classify_collated_data_size(collated_size)}); } -bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt) const { - return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt); +bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt, + td::uint64 collated_size) const { + return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) && + collated_data.fits(cls, collated_size); } td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const { @@ -729,20 +751,30 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S } int BlockLimitStatus::classify() const { - return limits.classify(estimate_block_size(), gas_used, cur_lt); + return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_size_estimate); } bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size())); + limits.bytes.fits(cls, estimate_block_size()) && limits.collated_data.fits(cls, collated_data_size_estimate)); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size(extra))); + limits.bytes.fits(cls, estimate_block_size(extra)) && + limits.collated_data.fits(cls, collated_data_size_estimate)); +} + +double BlockLimitStatus::load_fraction(unsigned cls) const { + if (cls >= ParamLimits::limits_cnt) { + return 0.0; + } + return std::max({(double)estimate_block_size() / (double)limits.bytes.limit(cls), + (double)gas_used / (double)limits.gas.limit(cls), + (double)collated_data_size_estimate / (double)limits.collated_data.limit(cls)}); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index 685005b48..c83fb6701 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -219,7 +219,7 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values td::uint32 max_bytes = 1 << 16; - td::uint32 max_msgs = 30; + td::uint32 max_msgs = 100; bool deserialize(vm::CellSlice& cs); ImportedMsgQueueLimits operator*(td::uint32 x) const { return {max_bytes * x, max_msgs * x}; @@ -242,6 +242,9 @@ struct ParamLimits { td::uint32 hard() const { return limits_[3]; } + td::uint32 limit(unsigned cls) const { + return limits_[cls]; + } bool compute_medium_limit() { limits_[2] = soft() + ((hard() - soft()) >> 1); return true; @@ -261,15 +264,17 @@ struct ParamLimits { }; struct BlockLimits { - ParamLimits bytes, gas, lt_delta; + ParamLimits bytes, gas, lt_delta, collated_data; ton::LogicalTime start_lt{0}; + ImportedMsgQueueLimits imported_msg_queue; const vm::CellUsageTree* usage_tree{nullptr}; bool deserialize(vm::CellSlice& cs); int classify_size(td::uint64 size) const; int classify_gas(td::uint64 gas) const; int classify_lt(ton::LogicalTime lt) const; - int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; - bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; + int classify_collated_data_size(td::uint64 size) const; + int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; + bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; }; struct BlockLimitStatus { @@ -278,6 +283,7 @@ struct BlockLimitStatus { td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; unsigned accounts{}, transactions{}, extra_out_msgs{}; + td::uint64 collated_data_size_estimate = 0; unsigned public_library_diff{}; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { @@ -289,12 +295,14 @@ struct BlockLimitStatus { gas_used = 0; extra_out_msgs = 0; public_library_diff = 0; + collated_data_size_estimate = 0; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; bool fits(unsigned cls) const; bool would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra = nullptr) const; + double load_fraction(unsigned cls) const; bool add_cell(Ref cell) { st_stat.add_cell(std::move(cell)); return true; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4a8bbc065..03b92cd2b 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -125,7 +125,7 @@ currencies$_ grams:Grams other:ExtraCurrencyCollection // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddressInt dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; @@ -134,19 +134,19 @@ ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; tick_tock$_ tick:Bool tock:Bool = TickTock; -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(Maybe ^Cell) = StateInit; // StateInitWithLibs is used to validate sent and received messages -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) = StateInitWithLibs; @@ -162,6 +162,16 @@ message$_ {X:Type} info:CommonMsgInfoRelaxed _ (Message Any) = MessageAny; + +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; +new_bounce_body#fffffffe + original_body:^Cell + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; + // interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; @@ -246,14 +256,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; -// -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - public_cells:(VarUInteger 7) = StorageUsed; -storage_used_short$_ cells:(VarUInteger 7) - bits:(VarUInteger 7) = StorageUsedShort; +storage_extra_none$000 = StorageExtraInfo; +storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; -storage_info$_ used:StorageUsed last_paid:uint32 +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed; + +storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32 due_payment:(Maybe Grams) = StorageInfo; account_none$0 = Account; @@ -273,14 +282,6 @@ acc_state_frozen$01 = AccountStatus; acc_state_active$10 = AccountStatus; acc_state_nonexist$11 = AccountStatus; -/* duplicates -tick_tock$_ tick:Bool tock:Bool = TickTock; - -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) - code:(Maybe ^Cell) data:(Maybe ^Cell) - library:(Maybe ^Cell) = StateInit; -*/ - account_descr$_ account:^Account last_trans_hash:bits256 last_trans_lt:uint64 = ShardAccount; @@ -341,13 +342,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 - action_list_hash:bits256 tot_msg_size:StorageUsedShort + action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; tr_phase_bounce_negfunds$00 = TrBouncePhase; -tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort +tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsedShort +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; // trans_ord$0000 credit_first:Bool @@ -679,6 +680,7 @@ workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) split_merge_timings:WcSplitMergeTimings + persistent_state_split_depth:(## 8) { persistent_state_split_depth <= 63 } = WorkchainDescr; _ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12; @@ -724,9 +726,13 @@ config_gas_prices#_ GasLimitsPrices = ConfigParam 21; param_limits#c3 underload:# soft_limit:# { underload <= soft_limit } hard_limit:# { soft_limit <= hard_limit } = ParamLimits; +imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits; block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits = BlockLimits; - +block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits + collated_data:ParamLimits imported_msg_queue:ImportedMsgQueueLimits + = BlockLimits; + config_mc_block_limits#_ BlockLimits = ConfigParam 22; config_block_limits#_ BlockLimits = ConfigParam 23; @@ -800,8 +806,9 @@ _ MisbehaviourPunishmentConfig = ConfigParam 40; size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 - max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; + max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_mc_acc_state_cells:uint32 + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 + max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] @@ -849,6 +856,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; +account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 431a03fec..c3c9085da 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -492,7 +492,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() + " with incorrect root hash"); } - TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks)); + TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, from, block::ConfigInfo::needPrevBlocks)); if (!config->check_old_mc_block_id(to, true)) { return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " + from.to_str() + " using the presented Merkle proof of masterchain state"); diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index c8c8b970d..a32564a21 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -94,12 +94,12 @@ typedef td::BitArray<256> hash_t; struct SmcDescr { hash_t addr; - int split_depth; + int fixed_prefix_length; bool preinit_only; td::RefInt256 gram_balance; Ref state_init; // StateInit Ref account; // Account - SmcDescr(const hash_t& _addr) : addr(_addr), split_depth(0), preinit_only(false) { + SmcDescr(const hash_t& _addr) : addr(_addr), fixed_prefix_length(0), preinit_only(false) { } }; @@ -123,7 +123,7 @@ vm::Dictionary config_dict{32}; ton::UnixTime now; bool set_config_smc(const SmcDescr& smc) { - if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.split_depth) { + if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.fixed_prefix_length) { return false; } vm::CellSlice cs = load_cell_slice(smc.state_init); @@ -221,7 +221,7 @@ bool add_public_library(hash_t lib_addr, hash_t smc_addr, Ref lib_root } td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, Ref data, - Ref library, td::RefInt256 balance, int special, int split_depth, + Ref library, td::RefInt256 balance, int special, int fixed_prefix_length, int mode) { if (is_empty_cell(code)) { code.clear(); @@ -238,12 +238,12 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R THRERR("not a valid library collection"); } vm::CellBuilder cb; - if (!split_depth) { + if (!fixed_prefix_length) { PDO(cb.store_long_bool(0, 1)); } else { - PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(split_depth, 5)); + PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(fixed_prefix_length, 5)); } - THRERR("invalid split_depth for a smart contract"); + THRERR("invalid fixed_prefix_length for a smart contract"); if (!special) { PDO(cb.store_long_bool(0, 1)); } else { @@ -287,7 +287,7 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R auto ins = smart_contracts.emplace(addr, addr); assert(ins.second); SmcDescr& smc = ins.first->second; - smc.split_depth = split_depth; + smc.fixed_prefix_length = fixed_prefix_length; smc.preinit_only = (mode == 1); smc.gram_balance = balance; total_smc_balance += balance; @@ -328,20 +328,21 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R ctor = 2; // addr_std$10 } PDO(cb.store_long_bool(ctor, 2)); // addr_std$10 or addr_var$11 - if (split_depth) { + if (fixed_prefix_length) { PDO(cb.store_long_bool(1, 1) // just$1 - && cb.store_ulong_rchk_bool(split_depth, 5) // depth:(## 5) - && cb.store_bits_bool(addr.cbits(), split_depth)); // rewrite pfx:(depth * Bit) + && cb.store_ulong_rchk_bool(fixed_prefix_length, 5) // depth:(## 5) + && cb.store_bits_bool(addr.cbits(), fixed_prefix_length)); // rewrite pfx:(depth * Bit) } else { PDO(cb.store_long_bool(0, 1)); // nothing$0 } PDO(cb.store_long_rchk_bool(workchain_id, ctor == 2 ? 8 : 32) && cb.store_bits_bool(addr.cbits(), 256)); THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract"); // storage_stat:StorageInfo -> storage_stat.used:StorageUsed - PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7) + PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) + && block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7) THRERR("Cannot serialize used:StorageUsed of the new smart contract"); + PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo + THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract"); PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams) PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage THRERR("Cannot create Account of the new smart contract"); @@ -514,7 +515,7 @@ Ref create_state() { // data (cell) // library (cell) // balance (int) -// split_depth (int 0..32) +// fixed_prefix_length (int 0..32) // special (int 0..3, +2 = tick, +1 = tock) // [ address (uint256) ] // mode (0 = compute address only, 1 = create uninit, 2 = create complete; +4 = with specified address) @@ -536,7 +537,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { if (special && workchain_id != wc_master) { throw fift::IntError{"cannot create special smartcontracts outside of the masterchain"}; } - int split_depth = stack.pop_smallint_range(32); + int fixed_prefix_length = stack.pop_smallint_range(32); td::RefInt256 balance = stack.pop_int_finite(); if (sgn(balance) < 0) { throw fift::IntError{"initial balance of a smartcontract cannot be negative"}; @@ -548,7 +549,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { Ref data = stack.pop_cell(); Ref code = stack.pop_cell(); td::RefInt256 addr = create_smartcontract(std::move(spec_addr), std::move(code), std::move(data), std::move(library), - std::move(balance), special, split_depth, mode); + std::move(balance), special, fixed_prefix_length, mode); if (addr.is_null()) { throw fift::IntError{"internal error while creating smartcontract"}; } diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0f019b068..d9a67b0be 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -38,6 +38,7 @@ #include "openssl/digest.hpp" #include #include +#include namespace block { using namespace std::literals::string_literals; @@ -91,22 +92,16 @@ td::Result> Config::extract_from_state(Ref mc_ return unpack_config(std::move(extra.config), mode); } -td::Result> ConfigInfo::extract_config(std::shared_ptr static_boc, - int mode) { - TRY_RESULT(rc, static_boc->get_root_count()); - if (rc != 1) { - return td::Status::Error(-668, "Masterchain state BoC is invalid"); - } - TRY_RESULT(root, static_boc->get_root_cell(0)); - return extract_config(std::move(root), mode); -} - -td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode) { +td::Result> ConfigInfo::extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode) { if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; TRY_STATUS(config->unpack_wrapped()); + if (!config->set_block_id_ext(mc_block_id)) { + return td::Status::Error("failed to set mc block id"); + } return std::move(config); } @@ -253,7 +248,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), true); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -486,15 +481,79 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +class ValidatorSetCache { +public: + ValidatorSetCache() { + cache_.reserve(MAX_CACHE_SIZE + 1); + } + + std::shared_ptr get(const vm::CellHash& hash) { + std::lock_guard lock{mutex_}; + auto it = cache_.find(hash); + if (it == cache_.end()) { + return {}; + } + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + return entry->value; + } + + void set(const vm::CellHash& hash, std::shared_ptr vset) { + std::lock_guard lock{mutex_}; + std::unique_ptr& entry = cache_[hash]; + if (entry == nullptr) { + entry = std::make_unique(hash, std::move(vset)); + } else { + entry->value = std::move(vset); + entry->remove(); + } + lru_.put(entry.get()); + if (cache_.size() > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(to_remove->key); + } + } + +private: + std::mutex mutex_; + + struct CacheEntry : td::ListNode { + explicit CacheEntry(vm::CellHash key, std::shared_ptr value) : key(key), value(std::move(value)) { + } + vm::CellHash key; + std::shared_ptr value; + }; + td::HashMap> cache_; + td::ListNode lru_; + + static constexpr size_t MAX_CACHE_SIZE = 100; +}; + +td::Result> Config::unpack_validator_set(Ref vset_root, bool use_cache) { if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + TRY_RESULT(loaded_root, vset_root->load_cell()); + if (!loaded_root.tree_node.empty()) { + // Do not use cache during Merkle proof generation + use_cache = false; + } + static ValidatorSetCache cache; + if (use_cache) { + auto result = cache.get(vset_root->get_hash()); + if (result) { + return result; + } + } + gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { gen::ValidatorSet::Record_validators rec0; - if (!tlb::unpack_cell(std::move(vset_root), rec0)) { + if (!tlb::unpack_cell(vset_root, rec0)) { return td::Status::Error("validator set is invalid"); } rec.utime_since = rec0.utime_since; @@ -515,39 +574,58 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); - for (int i = 0; i < rec.total; i++) { - key_buffer.store_ulong(i); - auto descr_cs = dict.lookup(key_buffer.bits(), 16); - if (descr_cs.is_null()) { - return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); - } + auto ptr = std::make_shared(rec.utime_since, rec.utime_until, rec.total, rec.main); + + std::vector seen_keys(rec.total); + td::Status error; + + auto validator_set_check_fn = [&](Ref descr_cs, td::ConstBitPtr key, int n) -> bool { + int i = static_cast(key.get_uint(n)); + CHECK(i >= 0 && i < rec.total && !seen_keys[i]); + seen_keys[i] = true; + gen::ValidatorDescr::Record_validator_addr descr; if (!tlb::csr_unpack(descr_cs, descr)) { descr.adnl_addr.set_zero(); if (!(gen::t_ValidatorDescr.unpack_validator(descr_cs.write(), descr.public_key, descr.weight) && descr_cs->empty_ext())) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has an invalid ValidatorDescr record in the validator set dictionary"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has an invalid ValidatorDescr record in the validator set dictionary"); + return false; } } gen::SigPubKey::Record sig_pubkey; if (!tlb::csr_unpack(std::move(descr.public_key), sig_pubkey)) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has no public key or its public key is in unsupported format"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has no public key or its public key is in unsupported format"); + return false; } if (!descr.weight) { - return td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + error = td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + return false; } if (descr.weight > ~(ptr->total_weight)) { - return td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + error = td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + return false; } ptr->list.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; + return true; + }; + + if (!dict.check_for_each(validator_set_check_fn)) { + CHECK(error.is_error()); + return error; + } + if (std::find(seen_keys.begin(), seen_keys.end(), false) != seen_keys.end()) { + return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); } if (rec.total_weight && rec.total_weight != ptr->total_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + if (use_cache) { + cache.set(vset_root->get_hash(), ptr); + } return std::move(ptr); } @@ -1956,11 +2034,13 @@ td::Result Config::do_get_size_limits_config(td::Ref& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain); }; @@ -394,10 +398,12 @@ struct SizeLimitsConfig { td::uint16 max_vm_data_depth = 512; ExtMsgLimits ext_msg_limits; td::uint32 max_acc_state_cells = 1 << 16; - td::uint32 max_acc_state_bits = (1 << 16) * 1023; + td::uint32 max_mc_acc_state_cells = 1 << 11; // enabled in global version 12 td::uint32 max_acc_public_libraries = 256; td::uint32 defer_out_queue_size_limit = 256; td::uint32 max_msg_extra_currencies = 2; + td::uint32 max_acc_fixed_prefix_length = 8; + td::uint32 acc_state_cells_for_storage_dict = 26; }; struct CatchainValidatorsConfig { @@ -433,6 +439,8 @@ struct WorkchainInfo : public td::CntObject { unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future + td::uint32 persistent_state_split_depth = 0; + bool is_valid() const { return workchain != ton::workchainInvalid; } @@ -539,6 +547,11 @@ struct PrecompiledContractsConfig { td::optional get_contract(td::Bits256 code_hash) const; }; +struct CollatorNodeDescr { + ton::ShardIdFull shard; + ton::NodeIdShort adnl_id; +}; + class Config { enum { default_mc_catchain_lifetime = 200, @@ -559,7 +572,7 @@ class Config { td::BitArray<256> config_addr; Ref config_root; std::unique_ptr config_dict; - std::unique_ptr cur_validators_; + std::shared_ptr cur_validators_; std::unique_ptr workchains_dict_; WorkchainSet workchains_; int version_{-1}; @@ -623,7 +636,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; @@ -643,8 +656,8 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - const ValidatorSet* get_cur_validator_set() const { - return cur_validators_.get(); + std::shared_ptr const& get_cur_validator_set() const& { + return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; ton::ValidatorSessionConfig get_consensus_config() const; @@ -763,9 +776,8 @@ class ConfigInfo : public Config, public ShardConfig { std::vector compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno* cc_seqno_delta = nullptr) const; td::Result> get_prev_blocks_info() const; - static td::Result> extract_config(std::shared_ptr static_boc, - int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0); + static td::Result> extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode = 0); private: ConfigInfo(Ref mc_state_root, int _mode = 0); diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 7d258cfe7..74b5845e7 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -134,31 +134,22 @@ bool OutputQueueMerger::MsgKeyValue::split(MsgKeyValue& second) { return true; } -bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { +void OutputQueueMerger::add_root(int src, Ref outmsg_root, td::int32 msg_limit) { if (outmsg_root.is_null()) { - return true; + return; } auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); } - return true; -} - -OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) - : queue_for(_queue_for), neighbors(std::move(_neighbors)), eof(false), failed(false) { - init(); -} - -OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) - : queue_for(_queue_for), eof(false), failed(false) { - for (auto& nb : _neighbors) { - neighbors.emplace_back(nb.top_block_id(), nb.outmsg_root, nb.is_disabled()); + if ((int)src_remaining_msgs_.size() < src + 1) { + src_remaining_msgs_.resize(src + 1); } - init(); + src_remaining_msgs_[src] = msg_limit; } -void OutputQueueMerger::init() { +OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors) + : eof(false), failed(false) { common_pfx.bits().store_int(queue_for.workchain, 32); int l = queue_for.pfx_len(); td::bitstring::bits_store_long_top(common_pfx.bits() + 32, queue_for.shard, l); @@ -168,7 +159,7 @@ void OutputQueueMerger::init() { if (!neighbor.disabled_) { LOG(DEBUG) << "adding " << (neighbor.outmsg_root_.is_null() ? "" : "non-") << "empty output queue for neighbor #" << i << " (" << neighbor.block_id_.to_str() << ")"; - add_root(i++, neighbor.outmsg_root_); + add_root(i++, neighbor.outmsg_root_, neighbor.msg_limit_); } else { LOG(DEBUG) << "skipping output queue for disabled neighbor #" << i; i++; @@ -222,7 +213,18 @@ bool OutputQueueMerger::load() { heap.pop_back(); } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); - return true; + for (size_t i = orig_size; i < msg_list.size(); ++i) { + td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; + if (remaining != -1) { + if (remaining == 0) { + limit_exceeded = true; + } else { + --remaining; + } + } + msg_list[i]->limit_exceeded = limit_exceeded; + } + return msg_list.size() > orig_size; } } // namespace block diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index 07533f243..26deb7ee4 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -32,6 +32,7 @@ struct OutputQueueMerger { int source; int key_len{0}; td::BitArray key; + bool limit_exceeded{false}; MsgKeyValue() = default; MsgKeyValue(int src, Ref node); MsgKeyValue(td::ConstBitPtr key_pfx, int key_pfx_len, int src, Ref node); @@ -51,23 +52,22 @@ struct OutputQueueMerger { bool unpack_node(td::ConstBitPtr key_pfx, int key_pfx_len, Ref node); bool split(MsgKeyValue& second); }; + // + std::vector> msg_list; + + public: struct Neighbor { ton::BlockIdExt block_id_; td::Ref outmsg_root_; bool disabled_; + td::int32 msg_limit_; // -1 - unlimited Neighbor() = default; - Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false) - : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled) { + Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false, td::int32 msg_limit = -1) + : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) { } }; - // - ton::ShardIdFull queue_for; - std::vector> msg_list; - std::vector neighbors; - public: - OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); - OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); + OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors); bool is_eof() const { return eof; } @@ -80,10 +80,11 @@ struct OutputQueueMerger { int common_pfx_len; std::vector> heap; std::size_t pos{0}; + std::vector src_remaining_msgs_; bool eof; bool failed; - void init(); - bool add_root(int src, Ref outmsg_root); + bool limit_exceeded{false}; + void add_root(int src, Ref outmsg_root, td::int32 msg_limit); bool load(); }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 34d235114..2d65c7229 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -115,42 +115,42 @@ bool Account::set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr) { } /** - * Sets the split depth of the account. + * Sets the length of anycast prefix length in the account address. * - * @param new_split_depth The new split depth value to be set. + * @param new_length The new rewrite length. * - * @returns True if the split depth was successfully set, False otherwise. + * @returns True if the length was successfully set, False otherwise. */ -bool Account::set_split_depth(int new_split_depth) { - if (new_split_depth < 0 || new_split_depth > 30) { - return false; // invalid value for split_depth +bool Account::set_addr_rewrite_length(int new_length) { + if (new_length < 0 || new_length > 30) { + return false; // invalid value } - if (split_depth_set_) { - return split_depth_ == new_split_depth; + if (addr_rewrite_length_set) { + return addr_rewrite_length == new_length; } else { - split_depth_ = (unsigned char)new_split_depth; - split_depth_set_ = true; + addr_rewrite_length = (unsigned char)new_length; + addr_rewrite_length_set = true; return true; } } /** - * Checks if the given split depth is valid for the Account. + * Checks if the given addr rewrite length is valid for the Account. * - * @param split_depth The split depth to be checked. + * @param length The addr rewrite length to be checked. * - * @returns True if the split depth is valid, False otherwise. + * @returns True if the addr rewrite length is valid, False otherwise. */ -bool Account::check_split_depth(int split_depth) const { - return split_depth_set_ ? (split_depth == split_depth_) : (split_depth >= 0 && split_depth <= 30); +bool Account::check_addr_rewrite_length(int length) const { + return addr_rewrite_length_set ? (length == addr_rewrite_length) : (length >= 0 && length <= 30); } /** * Parses anycast data of the account address. * - * Initializes split_depth and addr_rewrite. + * Initializes addr_rewrite. * - * @param cs The cell slice containing partially-parsed account addressa. + * @param cs The cell slice containing partially-parsed account address. * * @returns True if parsing was successful, false otherwise. */ @@ -159,13 +159,13 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { if (t < 0) { return false; } else if (!t) { - return set_split_depth(0); + return set_addr_rewrite_length(0); } int depth; return cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) && depth // { depth >= 1 } && cs.fetch_bits_to(addr_rewrite.bits(), depth) // rewrite_pfx:(bits depth) - && set_split_depth(depth); + && set_addr_rewrite_length(depth); } /** @@ -176,12 +176,12 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { * @returns True if the anycast information was successfully stored, false otherwise. */ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { - if (!split_depth_set_ || !split_depth_) { + if (!addr_rewrite_length_set || !addr_rewrite_length) { return cb.store_bool_bool(false); } return cb.store_bool_bool(true) // just$1 - && cb.store_uint_leq(30, split_depth_) // depth:(#<= 30) - && cb.store_bits_bool(addr_rewrite.cbits(), split_depth_); // rewrite_pfx:(bits depth) + && cb.store_uint_leq(30, addr_rewrite_length) // depth:(#<= 30) + && cb.store_bits_bool(addr_rewrite.cbits(), addr_rewrite_length); // rewrite_pfx:(bits depth) } /** @@ -214,14 +214,14 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { if (workchain == ton::workchainInvalid) { workchain = new_wc; addr = addr_orig; - addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); - } else if (split_depth_) { + addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); + } else if (addr_rewrite_length) { ton::StdSmcAddress new_addr = addr_orig; - new_addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); + new_addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); if (new_addr != addr) { LOG(ERROR) << "error unpacking account " << workchain << ":" << addr.to_hex() << " : account header contains different address " << new_addr.to_hex() << " (with splitting depth " - << (int)split_depth_ << ")"; + << (int)addr_rewrite_length << ")"; return false; } } else if (addr != addr_orig) { @@ -235,7 +235,7 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { return false; } addr_rewrite = addr.bits(); // initialize all 32 bits of addr_rewrite - if (!split_depth_) { + if (!addr_rewrite_length) { my_addr_exact = my_addr; } return true; @@ -257,6 +257,12 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return false; } last_paid = info.last_paid; + if (info.storage_extra.write().fetch_long(3) == 1) { + info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force()); + } else { + storage_dict_hash = {}; + } + orig_storage_dict_hash = storage_dict_hash; if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -268,11 +274,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { due_payment = td::zero_refint(); } unsigned long long u = 0; - u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); - u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells); - LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits - << " public_cells=" << storage_stat.public_cells; + u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); + u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); + LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits; return (u != std::numeric_limits::max()); } @@ -280,7 +284,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { * Unpacks the state of an Account from a CellSlice. * * State is serialized using StateInit TLB-scheme. - * Initializes split_depth (from account state - StateInit) + * Initializes fixed_prefix_length (from account state - StateInit) * * @param cs The CellSlice containing the serialized state. * @@ -291,12 +295,9 @@ bool Account::unpack_state(vm::CellSlice& cs) { if (!tlb::unpack_exact(cs, state)) { return false; } - int sd = 0; - if (state.split_depth->size() == 6) { - sd = (int)state.split_depth->prefetch_ulong(6) - 32; - } - if (!set_split_depth(sd)) { - return false; + fixed_prefix_length = 0; + if (state.fixed_prefix_length->size() == 6) { + fixed_prefix_length = (int)state.fixed_prefix_length->prefetch_ulong(6) - 32; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -307,8 +308,8 @@ bool Account::unpack_state(vm::CellSlice& cs) { tock = z & 1; LOG(DEBUG) << "tick=" << tick << ", tock=" << tock; } - code = state.code->prefetch_ref(); - data = state.data->prefetch_ref(); + code = orig_code = state.code->prefetch_ref(); + data = orig_data = state.data->prefetch_ref(); library = orig_library = state.library->prefetch_ref(); return true; } @@ -363,23 +364,25 @@ bool Account::compute_my_addr(bool force) { /** * Computes the address of the Account. * + * Legacy (used only if global_version < 10). + * * @param tmp_addr A reference to the CellSlice for the result. - * @param split_depth The split depth for the address. - * @param orig_addr_rewrite Address prefox of length split_depth. + * @param fixed_prefix_length The fixed prefix length for the address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the address was successfully computed, false otherwise. */ -bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, +bool Account::recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const { - if (!split_depth && my_addr_exact.not_null()) { + if (!fixed_prefix_length && my_addr_exact.not_null()) { tmp_addr = my_addr_exact; return true; } - if (split_depth == split_depth_ && my_addr.not_null()) { + if (fixed_prefix_length == addr_rewrite_length && my_addr.not_null()) { tmp_addr = my_addr; return true; } - if (split_depth < 0 || split_depth > 30) { + if (fixed_prefix_length < 0 || fixed_prefix_length > 30) { return false; } vm::CellBuilder cb; @@ -387,13 +390,13 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, if (!cb.store_long_bool(std ? 2 : 3, 2)) { // addr_std$10 or addr_var$11 return false; } - if (!split_depth) { + if (!fixed_prefix_length) { if (!cb.store_bool_bool(false)) { // anycast:(Maybe Anycast) return false; } } else if (!(cb.store_bool_bool(true) // just$1 - && cb.store_long_bool(split_depth, 5) // depth:(#<= 30) - && cb.store_bits_bool(addr.bits(), split_depth))) { // rewrite_pfx:(bits depth) + && cb.store_long_bool(fixed_prefix_length, 5) // depth:(#<= 30) + && cb.store_bits_bool(addr.bits(), fixed_prefix_length))) { // rewrite_pfx:(bits depth) return false; } if (std) { @@ -405,26 +408,26 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, return false; } Ref cell; - return cb.store_bits_bool(orig_addr_rewrite, split_depth) // address:(bits addr_len) or bits256 - && cb.store_bits_bool(addr.bits() + split_depth, 256 - split_depth) && cb.finalize_to(cell) && + return cb.store_bits_bool(orig_addr_rewrite, fixed_prefix_length) // address:(bits addr_len) or bits256 + && cb.store_bits_bool(addr.bits() + fixed_prefix_length, 256 - fixed_prefix_length) && cb.finalize_to(cell) && (tmp_addr = vm::load_cell_slice_ref(std::move(cell))).not_null(); } /** * Sets address rewriting info for a newly-activated account. * - * @param split_depth The split depth for the account address. - * @param orig_addr_rewrite Address frepix of length split_depth. + * @param rewrite_length The fixed prefix length for the account address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the rewriting info was successfully set, false otherwise. */ -bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !set_split_depth(split_depth)) { +bool Account::init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite) { + if (addr_rewrite_length_set || !set_addr_rewrite_length(rewrite_length)) { return false; } addr_orig = addr; addr_rewrite = addr.bits(); - addr_orig.bits().copy_from(orig_addr_rewrite, split_depth); + addr_orig.bits().copy_from(orig_addr_rewrite, rewrite_length); return compute_my_addr(true); } @@ -480,7 +483,7 @@ bool Account::unpack(Ref shard_account, ton::UnixTime now, bool s case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; - forget_split_depth(); + forget_addr_rewrite_length(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) { last_trans_hash_.set_zero(); now_ = now; last_paid = 0; - storage_stat.clear(); + storage_used = {}; + orig_storage_dict_hash = storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -554,18 +558,127 @@ bool Account::init_new(ton::UnixTime now) { } state_hash = addr_orig; status = orig_status = acc_nonexist; - split_depth_set_ = false; + addr_rewrite_length_set = false; return true; } /** - * Resets the split depth of the account. + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as CellSlice. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return result; +} + +/** + * Computes storage dict of the account from scratch. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @returns Root of the dictionary, or Error + */ +td::Result> Account::compute_account_storage_dict() const { + if (storage.is_null()) { + return td::Status::Error("cannot compute storage dict: empty storage"); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot compute storage dict: storage_dict_hash is not set"); + } + AccountStorageStat stat; + auto storage_for_stat = storage_without_extra_currencies(storage); + if (storage_for_stat.is_null()) { + return td::Status::Error("cannot compute storage dict: invalid storage"); + } + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); + // Root of AccountStorage is not counted in AccountStorageStat + td::uint64 expected_cells = stat.get_total_cells() + 1; + td::uint64 expected_bits = stat.get_total_bits() + storage->size(); + if (expected_cells != storage_used.cells || expected_bits != storage_used.bits) { + return td::Status::Error(PSTRING() << "invalid storage_used: computed cells=" << expected_cells + << " bits=" << expected_bits << ", found cells" << storage_used.cells + << " bits=" << storage_used.bits); + } + TRY_RESULT(root_hash, stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); + } + return stat.get_dict_root(); +} + +/** + * Initializes account_storage_stat of the account using the existing dict_root. + * This is not strictly necessary, as the storage stat is recomputed in Transaction. + * However, it can be used to optimize cell usage. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @param dict_root Root of the storage dictionary. * - * @returns True if the split depth was successfully reset, false otherwise. + * @returns Status of the operation. */ -bool Account::forget_split_depth() { - split_depth_set_ = false; - split_depth_ = 0; +td::Status Account::init_account_storage_stat(Ref dict_root) { + if (storage.is_null()) { + if (dict_root.not_null()) { + return td::Status::Error("storage is null, but dict_root is not null"); + } + account_storage_stat = {}; + return td::Status::OK(); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot init storage dict: storage_dict_hash is not set"); + } + // Root of AccountStorage is not counted in AccountStorageStat + if (storage_used.cells < 1 || storage_used.bits < storage->size()) { + return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells + << " bits=" << storage_used.bits << " storage_root_bits=" << storage->size()); + } + AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, + storage_used.bits - storage->size()); + TRY_RESULT(root_hash, new_stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); + } + account_storage_stat = std::move(new_stat); + return td::Status::OK(); +} + +/** + * Resets the fixed prefix length of the account. + * + * @returns True if the fixed prefix length was successfully reset, false otherwise. + */ +bool Account::forget_addr_rewrite_length() { + addr_rewrite_length_set = false; + addr_rewrite_length = 0; addr_orig = addr; my_addr = my_addr_exact; addr_rewrite = addr.bits(); @@ -583,9 +696,10 @@ bool Account::deactivate() { } // forget special (tick/tock) info tick = tock = false; + fixed_prefix_length = 0; if (status == acc_nonexist || status == acc_uninit) { - // forget split depth and address rewriting info - forget_split_depth(); + // forget fixed prefix length and address rewriting info + forget_addr_rewrite_length(); // forget specific state hash for deleted or uninitialized accounts (revert to addr) state_hash = addr; } @@ -617,12 +731,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { * @param payment The total sum to be updated. * @param delta The time delta for which the payment is calculated. * @param prices The storage prices. - * @param storage Account storage statistics. + * @param storage_used Account storage statistics. * @param is_mc A flag indicating whether the account is in the masterchain. */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, - const vm::CellStorageStat& storage, bool is_mc) { - td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; + const StorageUsed& storage_used, bool is_mc) { + td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits}; if (is_mc) { // storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price; c.mul_short(prices.mc_cell_price); @@ -643,7 +757,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * * @param now The current Unix time. * @param pricing The vector of storage prices. - * @param storage_stat Account storage statistics. + * @param storage_used Account storage statistics. * @param last_paid The Unix time when the last payment was made. * @param is_special A flag indicating if the account is special. * @param is_masterchain A flag indicating if the account is in the masterchain. @@ -651,7 +765,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * @returns The computed storage fees as RefInt256. */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { return td::zero_refint(); @@ -669,7 +783,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now); if (upto < valid_until) { assert(upto >= pricing[i].valid_since); - add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain); + add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain); } upto = valid_until; } @@ -685,7 +799,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: * @returns The computed storage fees as RefInt256. */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { - return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); + return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain()); } namespace transaction { @@ -706,6 +820,7 @@ Transaction::Transaction(const Account& _account, int ttype, ton::LogicalTime re , is_first(_account.transactions.empty()) , new_tick(_account.tick) , new_tock(_account.tock) + , new_fixed_prefix_length(_account.fixed_prefix_length) , now(_now) , account(_account) , my_addr(_account.my_addr) @@ -746,42 +861,61 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* }; } auto cs = vm::load_cell_slice(in_msg); - int tag = block::gen::t_CommonMsgInfo.get_tag(cs); - Ref src_addr, dest_addr; + int tag = gen::t_CommonMsgInfo.get_tag(cs); switch (tag) { - case block::gen::CommonMsgInfo::int_msg_info: { - block::gen::CommonMsgInfo::Record_int_msg_info info; - if (!(tlb::unpack(cs, info) && msg_balance_remaining.unpack(std::move(info.value)))) { + case gen::CommonMsgInfo::int_msg_info: { + if (!(tlb::unpack(cs, in_msg_info) && msg_balance_remaining.unpack(in_msg_info.value))) { return false; } - if (info.ihr_disabled && ihr_delivered) { + if (in_msg_info.ihr_disabled && ihr_delivered) { return false; } - bounce_enabled = info.bounce; - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(std::move(info.ihr_fee)); + td::RefInt256 ihr_fee; + if (cfg->global_version >= 12) { + ihr_fee = td::zero_refint(); + td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + new_bounce_format = extra_flags->get_bit(0); + new_bounce_format_full_body = extra_flags->get_bit(1); + } else { + // Legacy: extra_flags was previously ihr_fee + ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + } if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { in_fwd_fee = td::zero_refint(); msg_balance_remaining += std::move(ihr_fee); } - if (info.created_lt >= start_lt) { - start_lt = info.created_lt + 1; + if (in_msg_info.created_lt >= start_lt) { + start_lt = in_msg_info.created_lt + 1; end_lt = start_lt + 1; } // ... break; } - case block::gen::CommonMsgInfo::ext_in_msg_info: { - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + case gen::CommonMsgInfo::ext_in_msg_info: { + gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack(cs, info)) { return false; } - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + in_msg_info.ihr_disabled = in_msg_info.bounce = in_msg_info.bounced = false; + in_msg_info.src = info.src; + in_msg_info.dest = info.dest; + in_msg_info.created_at = in_msg_info.created_lt = 0; + if (cfg->disable_anycast) { + // Check that dest is addr_std without anycast + gen::MsgAddressInt::Record_addr_std rec; + if (!gen::csr_unpack(info.dest, rec)) { + LOG(DEBUG) << "destination address of the external message is not a valid addr_std"; + return false; + } + if (rec.anycast->size() > 1) { + LOG(DEBUG) << "destination address of the external message is an anycast address"; + return false; + } + } in_msg_type = 2; in_msg_extern = true; // compute forwarding fees for this external message @@ -1152,7 +1286,7 @@ namespace transaction { * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction * * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is - * not enough to clean up old queires, thus locking funds inside. + * not enough to clean up old queries, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu * It was proposed to validators to increase gas limit for this account to 70M for a limited amount @@ -1382,7 +1516,7 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { // The only context where PrevBlocksInfo (13 parameter of c7) is null is inside emulator // where it need to be set via transaction_emulator_set_prev_blocks_info (see emulator/emulator-extern.cpp) // Inside validator, collator and liteserver checking external message contexts - // prev_blocks_info is always not null, since get_prev_blocks_info() + // prev_blocks_info is always not null, since get_prev_blocks_info() // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223) tuple.push_back(vm::StackEntry::maybe(cfg.prev_blocks_info)); } @@ -1393,11 +1527,63 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { ? vm::StackEntry(td::make_refint(compute_phase->precompiled_gas_usage.value())) : vm::StackEntry()); // precompiled_gas_usage:Integer } + if (cfg.global_version >= 11) { + // in_msg_params:[...] + tuple.push_back(prepare_in_msg_params_tuple(trans_type == tr_ord ? &in_msg_info : nullptr, in_msg_state, + msg_balance_remaining)); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); } +/** + * Prepares tuple with unpacked parameters of the inbound message (for the 17th element of c7). + * `info` is: + * - For internal messages - just int_msg_info of the message + * - For external messages - artificial int_msg_info based on ext_msg_info of the messages. + * - For tick-tock transactions and get methods - nullptr. + * + * @param info Pointer to the message info. + * @param state_init State init of the message (null if absent). + * @param msg_balance_remaining Remaining balance of the message (it's sometimes different from value in info). + * + * @returns Tuple with message parameters. + */ +Ref Transaction::prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining) { + std::vector in_msg_params(10); + if (info != nullptr) { + in_msg_params[0] = td::make_refint(info->bounce ? -1 : 0); // bounce + in_msg_params[1] = td::make_refint(info->bounced ? -1 : 0); // bounced + in_msg_params[2] = info->src; // src_addr + in_msg_params[3] = info->fwd_fee.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer(info->fwd_fee); // fwd_fee + in_msg_params[4] = td::make_refint(info->created_lt); // created_lt + in_msg_params[5] = td::make_refint(info->created_at); // created_at + auto value = info->value; + in_msg_params[6] = + info->value.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer_skip(value.write()); // original value + in_msg_params[7] = msg_balance_remaining.is_valid() ? msg_balance_remaining.grams : td::zero_refint(); // value + in_msg_params[8] = msg_balance_remaining.is_valid() ? vm::StackEntry::maybe(msg_balance_remaining.extra) + : vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry::maybe(state_init); // state_init + } else { + in_msg_params[0] = td::zero_refint(); // bounce + in_msg_params[1] = td::zero_refint(); // bounced + static Ref addr_none = vm::CellBuilder{}.store_zeroes(2).as_cellslice_ref(); + in_msg_params[2] = addr_none; // src_addr + in_msg_params[3] = td::zero_refint(); // fed_fee + in_msg_params[4] = td::zero_refint(); // created_lt + in_msg_params[5] = td::zero_refint(); // created_at + in_msg_params[6] = td::zero_refint(); // original value + in_msg_params[7] = td::zero_refint(); // value + in_msg_params[8] = vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry{}; // state_init + } + return td::make_cnt_ref>(std::move(in_msg_params)); +} + /** * Computes the number of output actions in a list. * @@ -1438,10 +1624,13 @@ bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, in_msg_library = state.library->prefetch_ref(); return true; } - if (state.split_depth->size() == 6) { - new_split_depth = (signed char)(state.split_depth->prefetch_ulong(6) - 32); + if (state.fixed_prefix_length->size() == 6) { + new_fixed_prefix_length = (signed char)(state.fixed_prefix_length->prefetch_ulong(6) - 32); } else { - new_split_depth = 0; + new_fixed_prefix_length = 0; + } + if (!cfg.disable_anycast) { + new_addr_rewrite_length = new_fixed_prefix_length; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -1460,7 +1649,7 @@ bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, if (forbid_public_libs) { size_limits.max_acc_public_libraries = 0; } - auto S = check_state_limits(size_limits, false); + auto S = check_state_limits(size_limits, cfg.global_version, false); if (S.is_error()) { LOG(DEBUG) << "Cannot unpack msg state: " << S.move_as_error(); new_code = old_code; @@ -1496,19 +1685,26 @@ std::vector> Transaction::compute_vm_libraries(const ComputePhaseC /** * Checks if the input message StateInit hash corresponds to the account address. * + * @param cfg The configuration for the compute phase. + * * @returns True if the input message state hash is valid, False otherwise. */ -bool Transaction::check_in_msg_state_hash() { +bool Transaction::check_in_msg_state_hash(const ComputePhaseConfig& cfg) { CHECK(in_msg_state.not_null()); - CHECK(new_split_depth >= 0 && new_split_depth < 32); + CHECK(new_fixed_prefix_length >= 0 && new_fixed_prefix_length < 32); td::Bits256 in_state_hash = in_msg_state->get_hash().bits(); - int d = new_split_depth; + int d = new_fixed_prefix_length; if ((in_state_hash.bits() + d).compare(account.addr.bits() + d, 256 - d)) { return false; } orig_addr_rewrite = in_state_hash.bits(); orig_addr_rewrite_set = true; - return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + return true; + } else { + return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + } } /** @@ -1523,12 +1719,12 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom ComputePhase& cp = *compute_phase; CHECK(cp.precompiled_gas_usage); td::uint64 gas_usage = cp.precompiled_gas_usage.value(); - td::Timer timer; + td::RealCpuTimer timer; auto result = impl.run(my_addr, now, start_lt, balance, new_data, *in_msg_body, in_msg, msg_balance_remaining, in_msg_extern, compute_vm_libraries(cfg), cfg.global_version, cfg.max_vm_data_depth, new_code, cfg.unpacked_config_tuple, due_payment.not_null() ? due_payment : td::zero_refint(), gas_usage); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); cp.vm_init_state_hash = td::Bits256::zero(); cp.exit_code = result.exit_code; cp.out_of_gas = false; @@ -1539,7 +1735,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.success = (cp.accepted && result.committed); LOG(INFO) << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted << " success=" << cp.success << " gas_used=" << gas_usage - << " time=" << elapsed << "s"; + << " time=" << time_tvm.real << "s cpu_time=" << time_tvm.cpu; if (cp.accepted & use_msg_state) { was_activated = true; acc_status = Account::acc_active; @@ -1547,7 +1743,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom if (cfg.with_vm_log) { cp.vm_log = PSTRING() << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted - << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << elapsed << "s"; + << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << time_tvm.real << "s"; } if (cp.success) { cp.new_data = impl.get_c4(); @@ -1639,16 +1835,24 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { use_msg_state = true; bool forbid_public_libs = acc_status == Account::acc_uninit && account.is_masterchain(); // Forbid for deploying, allow for unfreezing - if (!(unpack_msg_state(cfg, false, forbid_public_libs) && account.check_split_depth(new_split_depth))) { - LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state"; + if (!(unpack_msg_state(cfg, false, forbid_public_libs) && + account.check_addr_rewrite_length(new_fixed_prefix_length))) { + LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad fixed_prefix_length; cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } - if (acc_status == Account::acc_uninit && !check_in_msg_state_hash()) { + if (acc_status == Account::acc_uninit && !check_in_msg_state_hash(cfg)) { LOG(DEBUG) << "in_msg_state hash mismatch, cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } + if (cfg.disable_anycast && acc_status == Account::acc_uninit && + new_fixed_prefix_length > cfg.size_limits.max_acc_fixed_prefix_length) { + LOG(DEBUG) << "cannot init account state: too big fixed prefix length (" << new_fixed_prefix_length << ", max " + << cfg.size_limits.max_acc_fixed_prefix_length << ")"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } } else if (acc_status != Account::acc_active) { // no state, cannot perform transactions cp.skip_reason = in_msg_state.not_null() ? ComputePhase::sk_bad_state : ComputePhase::sk_no_state; @@ -1671,6 +1875,11 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { return true; } } + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + new_addr_rewrite_length = 0; + force_remove_anycast_address = true; + } td::optional precompiled; if (new_code.not_null() && trans_type == tr_ord) { @@ -1743,9 +1952,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(DEBUG) << "starting VM"; cp.vm_init_state_hash = vm.get_state_hash(); - td::Timer timer; + td::RealCpuTimer timer; cp.exit_code = ~vm.run(); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); LOG(DEBUG) << "VM terminated with exit code " << cp.exit_code; cp.out_of_gas = (cp.exit_code == ~(int)vm::Excno::out_of_gas); cp.vm_final_state_hash = vm.get_final_state_hash(cp.exit_code); @@ -1771,7 +1980,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success - << ", time=" << elapsed << "s"; + << ", time=" << time_tvm.real << "s, cpu_time=" << time_tvm.cpu; if (logger != nullptr) { cp.vm_log = logger->get_log(); } @@ -1810,6 +2019,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { << cfg.flat_gas_limit << "]; remaining balance=" << balance.to_str(); CHECK(td::sgn(balance.grams) >= 0); } + cp.vm_loaded_cells = vm.extract_loaded_cells(); return true; } @@ -1840,23 +2050,28 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.action_fine = td::zero_refint(); td::Ref old_code = new_code, old_data = new_data, old_library = new_library; - auto enforce_state_limits = [&]() { + // 1 - ok, 0 - limits exceeded, -1 - fatal error + auto enforce_state_limits = [&]() -> int { if (account.is_special) { - return true; + return 1; } - auto S = check_state_limits(cfg.size_limits); + auto S = check_state_limits(cfg.size_limits, cfg.global_version); if (S.is_error()) { + if (S.code() != AccountStorageStat::errorcode_limits_exceeded) { + LOG(ERROR) << "Account storage stat error: " << S.move_as_error(); + return -1; + } // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); - new_storage_stat.clear(); + new_account_storage_stat = {}; new_code = old_code; new_data = old_data; new_library = old_library; ap.result_code = 50; ap.state_exceeds_limits = true; - return false; + return 0; } - return true; + return 1; }; int n = 0; @@ -1966,8 +2181,10 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.no_funds = true; } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; - // This is required here because changes to libraries are applied even if actipn phase fails - enforce_state_limits(); + // This is required here because changes to libraries are applied even if action phase fails + if (enforce_state_limits() == -1) { + return false; + } if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); ap.total_action_fees = ap.action_fine; @@ -1989,7 +2206,20 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { new_code = ap.new_code; } new_data = compute_phase->new_data; // tentative persistent data update applied - if (!enforce_state_limits()) { + int res = enforce_state_limits(); + if (res == -1) { + return false; + } + if (res == 0) { + if (cfg.extra_currency_v2) { + end_lt = ap.end_lt = start_lt + 1; + if (cfg.action_fine_enabled) { + ap.action_fine = std::min(ap.action_fine, balance.grams); + ap.total_action_fees = ap.action_fine; + balance.grams -= ap.action_fine; + total_fees += ap.action_fine; + } + } return true; } @@ -2104,7 +2334,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex(); } new_library = std::move(dict).extract_root_cell(); - } catch (vm::VmError& vme) { + } catch (vm::VmError&) { return 42; } ap.spec_actions++; @@ -2237,11 +2467,12 @@ bool Transaction::check_replace_src_addr(Ref& src_addr) const { * @param dest_addr A reference to the destination address of the transaction. * @param cfg The configuration for the action phase. * @param is_mc A pointer to a boolean where it will be stored whether the destination is in the masterchain. + * @param allow_anycast Allow anycast the address. * * @returns True if the destination address is valid, false otherwise. */ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc) const { + bool* is_mc, bool allow_anycast) const { if (!dest_addr->prefetch_ulong(1)) { // all external addresses allowed if (is_mc) { @@ -2301,6 +2532,9 @@ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const A } } if (rec.anycast->size() > 1) { + if (!allow_anycast) { + return false; + } // destination address is an anycast vm::CellSlice cs{*rec.anycast}; int d = (int)cs.fetch_ulong(6) - 32; @@ -2452,8 +2686,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.disable_custom_fess) { fwd_fee = ihr_fee = td::zero_refint(); } else { - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + fwd_fee = tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = cfg.global_version >= 12 ? td::zero_refint() : tlb::t_Grams.as_integer(info.extra_flags); + } + if (cfg.disable_ihr_flag) { + info.ihr_disabled = true; } } // set created_at and created_lt to correct values @@ -2468,11 +2705,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return 35; // invalid source address } bool to_mc = false; - if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc)) { + if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc, !cfg.disable_anycast)) { LOG(DEBUG) << "invalid destination address in a proposed outbound message"; return check_skip_invalid(36); // invalid destination address } - if (cfg.extra_currency_v2) { + if (!ext_msg && cfg.extra_currency_v2) { CurrencyCollection value; if (!value.unpack(info.value)) { LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; @@ -2482,7 +2719,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " << cfg.size_limits.max_msg_extra_currencies << ")"; // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded - return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + return check_skip_invalid(44); // invalid value:CurrencyCollection : too many extra currencies } info.value = value.pack(); } @@ -2663,7 +2900,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; - return check_skip_invalid(41); // to many extra currencies + return check_skip_invalid(44); // to many extra currencies } Ref new_extra; @@ -2688,7 +2925,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // re-pack message value CHECK(req.pack_to(info.value)); CHECK(block::tlb::t_Grams.pack_integer(info.fwd_fee, fwd_fee_remain)); - CHECK(block::tlb::t_Grams.pack_integer(info.ihr_fee, ihr_fee)); + if (cfg.global_version < 12) { + CHECK(block::tlb::t_Grams.pack_integer(info.extra_flags, ihr_fee)); + } // serialize message CHECK(tlb::csr_pack(msg.info, info)); @@ -2828,13 +3067,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, LOG(DEBUG) << "cannot parse currency field in action_reserve_currency"; return -1; } + if (cfg.extra_currency_v2 && reserve.has_extra()) { + LOG(DEBUG) << "cannot reserve extra currencies"; + return -1; + } LOG(DEBUG) << "action_reserve_currency: mode=" << mode << ", reserve=" << reserve.to_str() << ", balance=" << ap.remaining_balance.to_str() << ", original balance=" << original_balance.to_str(); if (mode & 4) { if (mode & 8) { - reserve = original_balance - reserve; + if (cfg.extra_currency_v2) { + reserve.grams = original_balance.grams - reserve.grams; + } else { + reserve = original_balance - reserve; + } } else { - reserve += original_balance; + if (cfg.extra_currency_v2) { + reserve.grams += original_balance.grams; + } else { + reserve += original_balance; + } } } else if (mode & 8) { LOG(DEBUG) << "invalid reserve mode " << mode; @@ -2847,7 +3098,7 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, if (mode & 2) { if (cfg.reserve_extra_enabled) { if (!reserve.clamp(ap.remaining_balance)) { - LOG(DEBUG) << "failed to clamp reserve amount" << mode; + LOG(DEBUG) << "failed to clamp reserve amount " << mode; return -1; } } else { @@ -2868,7 +3119,11 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, newc.grams = ap.remaining_balance.grams - reserve.grams; if (mode & 1) { // leave only res_grams, reserve everything else - std::swap(newc, reserve); + if (cfg.extra_currency_v2) { + std::swap(newc.grams, reserve.grams); + } else { + std::swap(newc, reserve); + } } // set remaining_balance to new_grams and new_extra ap.remaining_balance = std::move(newc); @@ -2931,68 +3186,69 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * @param global_version Global version (ConfigParam 8). + * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. - * - If the state limits exceed the maximum allowed range, returns an error. + * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. + * - If an error occurred during storage stat calculation, returns other error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, int global_version, + bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { - if (a.is_null()) { - return b.is_null(); - } - if (b.is_null()) { - return false; - } - return a->get_hash() == b->get_hash(); + return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { return td::Status::OK(); } - vm::CellStorageStat storage_stat; - storage_stat.limit_cells = size_limits.max_acc_state_cells; - storage_stat.limit_bits = size_limits.max_acc_state_bits; + AccountStorageStat storage_stat; + if (is_account_stat && account.account_storage_stat) { + storage_stat = AccountStorageStat{&account.account_storage_stat.value()}; + } { TD_PERF_COUNTER(transaction_storage_stat_a); - td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); - } + td::RealCpuTimer timer; + SCOPE_EXIT { + LOG_IF(INFO, timer.elapsed_real() > 0.1) << "Compute used storage (1) took " << timer.elapsed_real() << "s"; + if (is_account_stat) { + time_storage_stat += timer.elapsed_both(); } - return td::Status::OK(); }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); - if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + if (is_account_stat && compute_phase) { + storage_stat.add_hint(compute_phase->vm_loaded_cells); + } + StorageStatCalculationContext context{is_account_stat}; + StorageStatCalculationContext::Guard guard{&context}; + if (is_account_stat) { + storage_stat_updates.push_back(new_code); + storage_stat_updates.push_back(new_data); + storage_stat_updates.push_back(new_library); + } + TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); + } + + td::uint32 max_cells = account.is_masterchain() && global_version >= 12 ? size_limits.max_mc_acc_state_cells + : size_limits.max_acc_state_cells; + if (storage_stat.get_total_cells() > max_cells) { + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + << " (max cells=" << max_cells << ")"); + } + if (account.is_masterchain() && !cell_equal(account.library, new_library)) { + auto libraries_count = get_public_libraries_count(new_library); + if (libraries_count > size_limits.max_acc_public_libraries) { + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "too many public libraries: " << libraries_count << " (max " + << size_limits.max_acc_public_libraries << ")"); } } - - if (acc_status == Account::acc_active) { - storage_stat.clear_limit(); - } else { - storage_stat.clear(); - } - td::Status res; - if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { - res = td::Status::Error(PSTRING() << "account state is too big"); - } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && - get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { - res = td::Status::Error("too many public libraries"); - } else { - res = td::Status::OK(); - } - if (update_storage_stat) { + if (is_account_stat) { // storage_stat will be reused in compute_state() - new_storage_stat = std::move(storage_stat); + new_account_storage_stat.value_force() = std::move(storage_stat); } - return res; + return td::Status::OK(); } /** @@ -3008,8 +3264,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } bounce_phase = std::make_unique(); BouncePhase& bp = *bounce_phase; - block::gen::Message::Record msg; - block::gen::CommonMsgInfo::Record_int_msg_info info; + gen::Message::Record msg; + gen::CommonMsgInfo::Record_int_msg_info info; auto cs = vm::load_cell_slice(in_msg); if (!(tlb::unpack(cs, info) && gen::t_Maybe_Either_StateInit_Ref_StateInit.skip(cs) && cs.have(1) && cs.have_refs((int)cs.prefetch_ulong(1)))) { @@ -3019,6 +3275,44 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (cs.fetch_ulong(1)) { cs = vm::load_cell_slice(cs.prefetch_ref()); } + + vm::CellBuilder body; + if (new_bounce_format) { + body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe + if (new_bounce_format_full_body) { // original_body:^Cell + body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); + } else { + body.store_ref(vm::CellBuilder().store_bits(in_msg_body->as_bitslice()).finalize_novm()); + } + body.store_ref(vm::CellBuilder() + .append_cellslice(in_msg_info.value) // value:CurrencyCollection + .store_long(in_msg_info.created_lt, 64) // created_lt:uint64 + .store_long(in_msg_info.created_at, 32) // created_at:uint32 + .finalize_novm()); // original_info:^NewBounceOriginalInfo + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(-compute_phase->skip_reason, 32); // exit_code:int32 + } else if (!compute_phase->success) { + body.store_long(1, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->exit_code, 32); // exit_code:int32 + } else { + body.store_long(2, 8); // bounced_by_phase:uint8 + body.store_long(action_phase->result_code, 32); // exit_code:int32 + } + // compute_phase:(Maybe NewBounceComputePhaseInfo) + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 1); + } else { + body.store_long(1, 1); + body.store_long(compute_phase->gas_used, 32); // gas_used:uint32 + body.store_long(compute_phase->vm_steps, 32); // vm_steps:uint32 + } + } else if (cfg.bounce_msg_body) { + int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); + body.store_long_bool(-1, 32); // 0xffffffff tag + body.append_bitslice(cs.prefetch_bits(body_bits)); // truncated message body + } + info.ihr_disabled = true; info.bounce = false; info.bounced = true; @@ -3034,7 +3328,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { // compute size of message vm::CellStorageStat sstat; // for message size // preliminary storage estimation of the resulting message - sstat.compute_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(body.get_refs()); bp.msg_bits = sstat.bits; bp.msg_cells = sstat.cells; // compute forwarding fees @@ -3070,26 +3365,17 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { && cb.append_cellslice_bool(info.src) // src:MsgAddressInt && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt && msg_balance.store(cb) // value:CurrencyCollection - && block::tlb::t_Grams.store_long(cb, 0) // ihr_fee:Grams + && block::tlb::t_Grams.store_long(cb, 0) // extra_flags:(VarUInteger 16) && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 && cb.store_long_bool(info.created_at, 32) // created_at:uint32 && cb.store_bool_bool(false)); // init:(Maybe ...) - if (cfg.bounce_msg_body) { - int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); - if (cb.remaining_bits() >= body_bits + 33u) { - CHECK(cb.store_bool_bool(false) // body:(Either X ^X) -> left X - && cb.store_long_bool(-1, 32) // int = -1 ("message type") - && cb.append_bitslice(cs.prefetch_bits(body_bits))); // truncated message body - } else { - vm::CellBuilder cb2; - CHECK(cb.store_bool_bool(true) // body:(Either X ^X) -> right ^X - && cb2.store_long_bool(-1, 32) // int = -1 ("message type") - && cb2.append_bitslice(cs.prefetch_bits(body_bits)) // truncated message body - && cb.store_builder_ref_bool(std::move(cb2))); // ^X - } + if (cb.can_extend_by(1 + body.size(), body.size_refs())) { + // body:(Either X ^X) -> left X + CHECK(cb.store_bool_bool(false) && cb.append_builder_bool(body)); } else { - CHECK(cb.store_bool_bool(false)); // body:(Either ..) + // body:(Either X ^X) -> right ^X + CHECK(cb.store_bool_bool(true) && cb.store_builder_ref_bool(std::move(body))); } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { @@ -3140,83 +3426,12 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. - * - * It succeeds if only root cell of AccountStorage is changed. - * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). - * - * @param old_stat The old storage statistics. - * @param old_cs The old AccountStorage. - * @param new_cell The new AccountStorage. - * - * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. - */ -static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, - td::Ref old_cs, - td::Ref new_cell) { - if (old_stat.cells == 0 || old_cs.is_null()) { - return {}; - } - vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell); - if (old_cs->size_refs() != new_cs.size_refs()) { - return {}; - } - for (unsigned i = 0; i < old_cs->size_refs(); ++i) { - if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) { - return {}; - } - } - if (old_stat.bits < old_cs->size()) { - return {}; - } - - vm::CellStorageStat new_stat; - new_stat.cells = old_stat.cells; - new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size(); - new_stat.public_cells = old_stat.public_cells; - return new_stat; -} - -/** - * Removes extra currencies dict from AccountStorage. - * - * This is used for computing account storage stats. - * - * @param storage_cs AccountStorage as CellSlice. - * - * @returns AccountStorage without extra currencies as Cell. - */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { - block::gen::AccountStorage::Record rec; - if (!block::gen::csr_unpack(storage_cs, rec)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - if (rec.balance->size_refs() > 0) { - block::gen::CurrencyCollection::Record balance; - if (!block::gen::csr_unpack(rec.balance, balance)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); - if (!block::gen::csr_pack(rec.balance, balance)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - } - td::Ref cell; - if (!block::gen::pack_cell(cell, rec)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - return cell; -} - namespace transaction { /** * Computes the new state of the account. * + * @param cfg The configuration for the serialization phase. + * * @returns True if the state computation is successful, false otherwise. */ bool Transaction::compute_state(const SerializeConfig& cfg) { @@ -3228,6 +3443,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { acc_status = Account::acc_nonexist; was_created = false; } + if (acc_status == Account::acc_deleted && !balance.is_zero()) { + acc_status = Account::acc_uninit; + } if (acc_status == Account::acc_nonexist || acc_status == Account::acc_deleted) { CHECK(balance.is_zero()); vm::CellBuilder cb; @@ -3240,13 +3458,14 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { && balance.store(cb)); // balance:CurrencyCollection int ticktock = new_tick * 2 + new_tock; unsigned si_pos = 0; + int fixed_prefix_length = cfg.disable_anycast ? new_fixed_prefix_length : account.addr_rewrite_length; if (acc_status == Account::acc_uninit) { CHECK(cb.store_long_bool(0, 2)); // account_uninit$00 = AccountState } else if (acc_status == Account::acc_frozen) { if (was_frozen) { vm::CellBuilder cb2; - CHECK(account.split_depth_ ? cb2.store_long_bool(account.split_depth_ + 32, 6) // _ ... = StateInit - : cb2.store_long_bool(0, 1)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb2.store_long_bool(fixed_prefix_length + 32, 6) // _ ... = StateInit + : cb2.store_long_bool(0, 1)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb2.store_long_bool(ticktock | 4, 3) : cb2.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb2.store_maybe_ref(new_code) && cb2.store_maybe_ref(new_data) && cb2.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3275,8 +3494,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { CHECK(acc_status == Account::acc_active && !was_frozen && !was_deleted); si_pos = cb.size_ext() + 1; - CHECK(account.split_depth_ ? cb.store_long_bool(account.split_depth_ + 96, 7) // account_active$1 _:StateInit - : cb.store_long_bool(2, 2)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb.store_long_bool(fixed_prefix_length + 96, 7) // account_active$1 _:StateInit + : cb.store_long_bool(2, 2)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb.store_long_bool(ticktock | 4, 3) : cb.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb.store_maybe_ref(new_code) && cb.store_maybe_ref(new_data) && cb.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3290,45 +3509,102 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { new_inner_state.clear(); } - vm::CellStorageStat& stats = new_storage_stat; + td::Ref old_storage_for_stat = account.storage; - td::Ref new_storage_for_stat = storage; + td::Ref new_storage_for_stat = new_storage; if (cfg.extra_currency_v2) { new_storage_for_stat = storage_without_extra_currencies(new_storage); if (new_storage_for_stat.is_null()) { return false; } if (old_storage_for_stat.not_null()) { - old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat); if (old_storage_for_stat.is_null()) { return false; } } + } else if (cfg.store_storage_dict_hash) { + LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false"; + return false; } - auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); - if (new_stats) { - stats = new_stats.unwrap(); + + bool storage_refs_changed = false; + if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) { + storage_refs_changed = true; } else { + for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) { + if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) { + storage_refs_changed = true; + break; + } + } + } + + bool store_storage_dict_hash = cfg.store_storage_dict_hash && !account.is_masterchain(); + if (storage_refs_changed || + (store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(new_storage_for_stat).ensure(); + if (!new_account_storage_stat && account.account_storage_stat) { + new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); + if (compute_phase) { + new_account_storage_stat.value().add_hint(compute_phase->vm_loaded_cells); + } + } + AccountStorageStat& stats = new_account_storage_stat.value_force(); + // Don't check Merkle depth and size here - they were checked in check_state_limits + auto roots = new_storage_for_stat->prefetch_all_refs(); + storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); + { + td::RealCpuTimer timer; + StorageStatCalculationContext context{true}; + StorageStatCalculationContext::Guard guard{&context}; + td::Status S = stats.replace_roots(roots); + time_storage_stat += timer.elapsed_both(); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + return false; + } + } + // Root of AccountStorage is not counted in AccountStorageStat + new_storage_used.cells = stats.get_total_cells() + 1; + new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); + if (store_storage_dict_hash && new_storage_used.cells >= cfg.size_limits.acc_state_cells_for_storage_dict) { + auto r_hash = stats.get_dict_hash(); + if (r_hash.is_error()) { + LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " + << r_hash.move_as_error(); + return false; + } + new_storage_dict_hash = r_hash.move_as_ok(); + } if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; + } + } else { + new_storage_used = account.storage_used; + new_storage_used.bits -= old_storage_for_stat->size(); + new_storage_used.bits += new_storage_for_stat->size(); + new_account_storage_stat = {}; + if (store_storage_dict_hash) { + new_storage_dict_hash = account.storage_dict_hash; } } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt - && block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7) - && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 + + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(cfg.disable_anycast ? my_addr : account.my_addr) // addr:MsgAddressInt + && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) + && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) + && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo + && (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256 + && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 if (due_payment.not_null() && td::sgn(due_payment) != 0) { CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment)); // due_payment:(Maybe Grams) } else { CHECK(cb.store_long_bool(0, 1)); } - CHECK(cb.append_data_cell_bool(std::move(storage))); + CHECK(cb.append_cellslice_bool(new_storage)); new_total_state = cb.finalize(); if (verbosity > 2) { FLOG(INFO) { @@ -3345,6 +3621,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { * * Updates root. * + * @param cfg The configuration for the serialization. + * * @returns True if the serialization is successful, False otherwise. */ bool Transaction::serialize(const SerializeConfig& cfg) { @@ -3676,19 +3954,29 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && + if (force_remove_anycast_address) { + CHECK(acc.forget_addr_rewrite_length()); + } else if (orig_addr_rewrite_set && new_addr_rewrite_length >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() - << " with split_depth=" << new_split_depth - << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_split_depth); - CHECK(acc.init_rewrite_addr(new_split_depth, orig_addr_rewrite.bits())); + << " with addr_rewrite_length=" << new_addr_rewrite_length + << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_addr_rewrite_length); + CHECK(acc.init_rewrite_addr(new_addr_rewrite_length, orig_addr_rewrite.bits())); } acc.status = (acc_status == Account::acc_deleted ? Account::acc_nonexist : acc_status); acc.last_trans_lt_ = start_lt; acc.last_trans_end_lt_ = end_lt; acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; - acc.storage_stat = new_storage_stat; + acc.storage_used = new_storage_used; + if (new_account_storage_stat) { + if (acc.account_storage_stat) { + acc.account_storage_stat.value().apply_child_stat(std::move(new_account_storage_stat.value())); + } else { + acc.account_storage_stat = std::move(new_account_storage_stat); + } + } + acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); acc.due_payment = std::move(due_payment); @@ -3705,6 +3993,7 @@ Ref Transaction::commit(Account& acc) { if (acc.status == Account::acc_active) { acc.tick = new_tick; acc.tock = new_tock; + acc.fixed_prefix_length = new_fixed_prefix_length; } else { CHECK(acc.deactivate()); } @@ -3905,6 +4194,7 @@ td::Status FetchConfigParams::fetch_config_params( compute_phase_cfg->size_limits = size_limits; compute_phase_cfg->precompiled_contracts = config.get_precompiled_contracts_config(); compute_phase_cfg->allow_external_unfreeze = compute_phase_cfg->global_version >= 8; + compute_phase_cfg->disable_anycast = config.get_global_version() >= 10; } { // compute action_phase_cfg @@ -3933,9 +4223,15 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + action_phase_cfg->disable_anycast = config.get_global_version() >= 10; + action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; + action_phase_cfg->global_version = config.get_global_version(); } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->disable_anycast = config.get_global_version() >= 10; + serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; + serialize_cfg->size_limits = size_limits; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 8e612e6a5..c4c13a558 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "account-storage-stat.h" #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/cells.h" @@ -30,6 +31,7 @@ #include "block/block.h" #include "block/mc-config.h" #include "precompiled-smc/PrecompiledSmartContract.h" +#include "block/block-auto.h" namespace block { using td::Ref; @@ -130,6 +132,7 @@ struct ComputePhaseConfig { PrecompiledContractsConfig precompiled_contracts; bool dont_run_precompiled_ = false; bool allow_external_unfreeze{false}; + bool disable_anycast{false}; ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); @@ -171,7 +174,10 @@ struct ActionPhaseConfig { bool disable_custom_fess{false}; bool reserve_extra_enabled{false}; bool extra_currency_v2{false}; + bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; + bool disable_anycast{false}; + int global_version = 0; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -179,6 +185,9 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool disable_anycast{false}; + bool store_storage_dict_hash{false}; + SizeLimitsConfig size_limits; }; struct CreditPhase { @@ -187,7 +196,7 @@ struct CreditPhase { }; struct ComputePhase { - enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended }; + enum { sk_none = 0, sk_no_state = 1, sk_bad_state = 2, sk_no_gas = 3, sk_suspended = 4 }; int skip_reason{sk_none}; bool success{false}; bool msg_state_used{false}; @@ -206,6 +215,7 @@ struct ComputePhase { Ref actions; std::string vm_log; td::optional precompiled_gas_usage; + td::HashSet vm_loaded_cells; }; struct ActionPhase { @@ -252,12 +262,13 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool split_depth_set_{false}; - unsigned char split_depth_{0}; + int fixed_prefix_length{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; - td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits + td::BitArray<32> addr_rewrite; // rewrite (anycast) data, addr_rewrite_length bits + bool addr_rewrite_length_set{false}; + unsigned char addr_rewrite_length{0}; ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info @@ -266,8 +277,13 @@ struct Account { ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; ton::LogicalTime block_lt; + ton::UnixTime last_paid; - vm::CellStorageStat storage_stat; + StorageUsed storage_used; + td::optional storage_dict_hash; + td::optional orig_storage_dict_hash; + td::optional account_storage_stat; + block::CurrencyCollection balance; td::RefInt256 due_payment; Ref orig_total_state; // ^Account @@ -275,22 +291,22 @@ struct Account { Ref storage; // AccountStorage Ref inner_state; // StateInit ton::Bits256 state_hash; // hash of StateInit for frozen accounts - Ref code, data, library, orig_library; + Ref code, data, library; + Ref orig_code, orig_data, orig_library; std::vector transactions; Account() = default; Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) { } - Account(ton::WorkchainId wc, td::ConstBitPtr _addr, int depth) - : split_depth_set_(true), split_depth_((unsigned char)depth), workchain(wc), addr(_addr) { - } block::CurrencyCollection get_balance() const { return balance; } bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); + td::Result> compute_account_storage_dict() const; + td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); - bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; + bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { return workchain == ton::masterchainId; @@ -306,10 +322,10 @@ struct Account { protected: friend struct transaction::Transaction; - bool set_split_depth(int split_depth); - bool check_split_depth(int split_depth) const; - bool forget_split_depth(); - bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); + bool set_addr_rewrite_length(int new_length); + bool check_addr_rewrite_length(int length) const; + bool forget_addr_rewrite_length(); + bool init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite); private: bool unpack_address(vm::CellSlice& addr_cs); @@ -341,12 +357,17 @@ struct Transaction { bool was_created{false}; bool bounce_enabled{false}; bool in_msg_extern{false}; + gen::CommonMsgInfo::Record_int_msg_info in_msg_info; + bool new_bounce_format{false}; + bool new_bounce_format_full_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; bool new_tick; bool new_tock; - signed char new_split_depth{-1}; + int new_fixed_prefix_length{-1}; + int new_addr_rewrite_length{-1}; + bool force_remove_anycast_address = false; ton::UnixTime now; int acc_status; int verbosity{3 * 0}; @@ -377,12 +398,16 @@ struct Transaction { std::unique_ptr compute_phase; std::unique_ptr action_phase; std::unique_ptr bounce_phase; - vm::CellStorageStat new_storage_stat; + StorageUsed new_storage_used; + td::optional new_account_storage_stat; + td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; + std::vector> storage_stat_updates; + td::RealCpuTimer::Time time_tvm, time_storage_stat; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); - bool check_in_msg_state_hash(); + bool check_in_msg_state_hash(const ComputePhaseConfig& cfg); bool prepare_storage_phase(const StoragePhaseConfig& cfg, bool force_collect = true, bool adjust_msg_value = false); bool prepare_credit_phase(); td::uint64 gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms); @@ -392,7 +417,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, int global_version, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); @@ -418,13 +443,18 @@ struct Transaction { int try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg); bool check_replace_src_addr(Ref& src_addr) const; bool check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc = nullptr) const; + bool* is_mc = nullptr, bool allow_anycast = true) const; bool serialize_storage_phase(vm::CellBuilder& cb); bool serialize_credit_phase(vm::CellBuilder& cb); bool serialize_compute_phase(vm::CellBuilder& cb); bool serialize_action_phase(vm::CellBuilder& cb); bool serialize_bounce_phase(vm::CellBuilder& cb); bool unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only = false, bool forbid_public_libs = false); + + public: + static Ref prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining); }; } // namespace transaction diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 52e57c9a8..3a6f33119 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -347,6 +347,9 @@ std::size_t bits_memscan_rev(ConstBitPtr bs, std::size_t bit_count, bool cmp_to) int bits_memcmp(const unsigned char* bs1, int bs1_offs, const unsigned char* bs2, int bs2_offs, std::size_t bit_count, std::size_t* same_upto) { if (!bit_count) { + if (same_upto) { + *same_upto = 0; + } return 0; } bs1 += (bs1_offs >> 3); diff --git a/crypto/ellcurve/Ed25519.cpp b/crypto/ellcurve/Ed25519.cpp deleted file mode 100644 index a38eb079e..000000000 --- a/crypto/ellcurve/Ed25519.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Ed25519.h" - -#include "td/utils/Random.h" - -namespace crypto { -namespace Ed25519 { - -bool all_bytes_same(const unsigned char *str, std::size_t size) { - unsigned char c = str[0]; - for (std::size_t i = 0; i < size; i++) { - if (str[i] != c) { - return false; - } - } - return true; -} - -void PublicKey::clear(void) { - if (inited != pk_empty) { - std::memset(pubkey, 0, pubkey_bytes); - PubKey.zeroize(); - PubKey_xz.zeroize(); - } - inited = pk_empty; -} - -PublicKey::PublicKey(const unsigned char pub_key[pubkey_bytes]) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(pub_key); -} - -PublicKey::PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(Pub_Key); -} - -bool PublicKey::import_public_key(const unsigned char pub_key[pubkey_bytes]) { - clear(); - if (all_bytes_same(pub_key, pubkey_bytes)) { - return false; - } - bool ok = false; - PubKey = ellcurve::Ed25519().import_point(pub_key, ok); - if (!ok) { - clear(); - return false; - } - std::memcpy(pubkey, pub_key, pubkey_bytes); - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - return true; -} - -bool PublicKey::import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) { - clear(); - if (!Pub_Key.is_valid()) { - return false; - } - PubKey = Pub_Key; - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - - if (!PubKey.export_point(pubkey)) { - clear(); - return false; - } - return true; -} - -bool PublicKey::export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const { - if (inited != pk_init) { - std::memset(pubkey_buffer, 0, pubkey_bytes); - return false; - } else { - std::memcpy(pubkey_buffer, pubkey, pubkey_bytes); - return true; - } -} - -bool PublicKey::check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size) { - if (inited != pk_init) { - return false; - } - unsigned char hash[64]; - { - digest::SHA512 hasher(signature, 32); - hasher.feed(pubkey, 32); - hasher.feed(message, msg_size); - hasher.extract(hash); - } - auto &E = ellcurve::Ed25519(); - const arith::Bignum &L = E.get_ell(); - arith::Bignum H, S; - S.import_lsb(signature + 32, 32); - H.import_lsb(hash, 64); - H %= L; - H = L - H; - auto sG = E.power_gen(S); - auto hA = E.power_point(PubKey, H); - auto pR1 = E.add_points(sG, hA); - unsigned char pR1_bytes[32]; - if (!pR1.export_point(pR1_bytes)) { - return false; - } - return !std::memcmp(pR1_bytes, signature, 32); -} - -// --------------------- -class PrivateKey; - -bool PrivateKey::random_private_key(bool strong) { - inited = false; - if (!prng::rand_gen().rand_bytes(privkey, privkey_bytes, strong)) { - clear(); - return false; - } - return process_private_key(); -} - -void PrivateKey::clear(void) { - std::memset(privkey, 0, privkey_bytes); - std::memset(priv_salt, 0, sizeof(priv_salt)); - priv_exp.clear(); - PubKey.clear(); - inited = false; -} - -bool PrivateKey::import_private_key(const unsigned char pk[privkey_bytes]) { - clear(); - if (all_bytes_same(pk, privkey_bytes)) { - return false; - } - std::memcpy(privkey, pk, privkey_bytes); - return process_private_key(); -} - -bool PrivateKey::export_private_key(unsigned char pk[privkey_bytes]) const { // careful! - if (!inited) { - std::memset(pk, 0, privkey_bytes); - return false; - } else { - std::memcpy(pk, privkey, privkey_bytes); - return true; - } -} - -bool PrivateKey::process_private_key() { - unsigned char buff[64]; - digest::hash_str(buff, privkey, privkey_bytes); - std::memcpy(priv_salt, buff + 32, 32); - buff[0] = (unsigned char)(buff[0] & -8); - buff[31] = (unsigned char)((buff[31] | 0x40) & ~0x80); - priv_exp.import_lsb(buff, 32); - PubKey = ellcurve::Ed25519().power_gen(priv_exp, true); // uniform - inited = PubKey.ok(); - if (!inited) { - clear(); - } - return inited; -} - -bool PrivateKey::compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub) { - if (!inited || !Pub.ok()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - // uniform power! - auto P = ellcurve::Curve25519().power_xz(Pub.get_point_xz(), priv_exp); - if (P.is_infty()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - P.export_point_u(secret); - return true; -} - -bool PrivateKey::compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]) { - PublicKey tempPubkey(temp_pub_key); - if (!tempPubkey.ok()) { - return false; - } - return compute_shared_secret(secret, tempPubkey); -} - -bool PrivateKey::sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size) { - if (!inited) { - std::memset(signature, 0, sign_bytes); - return false; - } - unsigned char r_bytes[64]; - digest::hash_two_str(r_bytes, priv_salt, 32, message, msg_size); - const arith::Bignum &L = ellcurve::Ed25519().get_ell(); - arith::Bignum eR; - eR.import_lsb(r_bytes, 64); - eR %= L; - std::memset(r_bytes, 0, sizeof(r_bytes)); - - // uniform power - auto pR = ellcurve::Ed25519().power_gen(eR, true); - - auto ok = pR.export_point(signature, true); - (void)ok; - assert(ok); - { - digest::SHA512 hasher(signature, 32); - hasher.feed(PubKey.get_pubkey_ptr(), 32); - hasher.feed(message, msg_size); - hasher.extract(r_bytes); - } - arith::Bignum S; - S.import_lsb(r_bytes, 64); - S %= L; - S *= priv_exp; - S += eR; - S %= L; - eR.clear(); - S.export_lsb(signature + 32, 32); - return true; -} - -// --------------------------------- -class TempKeyGenerator; - -unsigned char *TempKeyGenerator::get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand, - std::size_t rand_size) { // rand may be 0 - digest::SHA256 hasher(message, size); - hasher.feed(random_salt, salt_size); - if (rand && rand_size) { - hasher.feed(rand, rand_size); - } - if (!to) { - to = buffer; - } - hasher.extract(to); - //++ *((long *)random_salt); - return to; -} - -void TempKeyGenerator::create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand, std::size_t rand_size) { - pk.import_private_key(get_temp_private_key(buffer, message, size, rand, rand_size)); - std::memset(buffer, 0, privkey_bytes); -} - -bool TempKeyGenerator::create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], - unsigned char shared_secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, - std::size_t size, const unsigned char *rand, std::size_t rand_size) { - PrivateKey tmpPk; - create_temp_private_key(tmpPk, message, size, rand, rand_size); - return tmpPk.export_public_key(temp_pub_key) && tmpPk.compute_shared_secret(shared_secret, recipientPubKey); -} - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Ed25519.h b/crypto/ellcurve/Ed25519.h deleted file mode 100644 index 4b9e6348f..000000000 --- a/crypto/ellcurve/Ed25519.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include "ellcurve/Montgomery.h" -#include "ellcurve/TwEdwards.h" -#include "openssl/digest.hpp" -#include "openssl/rand.hpp" -#include -#include - -#include "td/utils/buffer.h" - -namespace crypto { -namespace Ed25519 { - -const int privkey_bytes = 32; -const int pubkey_bytes = 32; -const int sign_bytes = 64; -const int shared_secret_bytes = 32; - -class PublicKey { - enum { pk_empty, pk_xz, pk_init } inited; - unsigned char pubkey[pubkey_bytes]; - ellcurve::TwEdwardsCurve::SegrePoint PubKey; - ellcurve::MontgomeryCurve::PointXZ PubKey_xz; - - public: - PublicKey() : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - } - PublicKey(const unsigned char pub_key[pubkey_bytes]); - PublicKey(td::Slice pub_key) : PublicKey(pub_key.ubegin()) { - CHECK(pub_key.size() == pubkey_bytes); - } - PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - - bool import_public_key(const unsigned char pub_key[pubkey_bytes]); - bool import_public_key(td::Slice pub_key) { - CHECK(pub_key.size() == pubkey_bytes); - return import_public_key(pub_key.ubegin()); - } - bool import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - bool export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const; - bool export_public_key(td::MutableSlice pubk) const { - CHECK(pubk.size() == pubkey_bytes); - return export_public_key(pubk.ubegin()); - } - bool check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size); - bool check_message_signature(td::Slice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return check_message_signature(signature.ubegin(), message.ubegin(), message.size()); - } - - void clear(); - bool ok() const { - return inited == pk_init; - } - - const unsigned char *get_pubkey_ptr() const { - return inited == pk_init ? pubkey : 0; - } - const ellcurve::TwEdwardsCurve::SegrePoint &get_point() const { - return PubKey; - } - const ellcurve::MontgomeryCurve::PointXZ &get_point_xz() const { - return PubKey_xz; - } -}; - -class PrivateKey { - public: - struct priv_key_no_copy {}; - PrivateKey() : inited(false) { - memset(privkey, 0, privkey_bytes); - } - PrivateKey(const unsigned char pk[privkey_bytes]) : inited(false) { - memset(privkey, 0, privkey_bytes); - import_private_key(pk); - } - PrivateKey(td::Slice pk) : inited(false) { - CHECK(pk.size() == privkey_bytes); - memset(privkey, 0, privkey_bytes); - import_private_key(pk.ubegin()); - } - ~PrivateKey() { - clear(); - } - bool random_private_key(bool strong = false); - bool import_private_key(const unsigned char pk[privkey_bytes]); - bool import_private_key(td::Slice pk) { - CHECK(pk.size() == privkey_bytes); - return import_private_key(pk.ubegin()); - } - bool export_private_key(unsigned char pk[privkey_bytes]) const; // careful! - bool export_private_key(td::MutableSlice pk) const { // careful! - return export_private_key(pk.ubegin()); - } - bool export_public_key(unsigned char pubk[pubkey_bytes]) const { - return PubKey.export_public_key(pubk); - } - bool export_public_key(td::MutableSlice pubk) const { - return PubKey.export_public_key(pubk); - } - void clear(); - bool ok() const { - return inited; - } - - // used for EdDSA (sign) - bool sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size); - bool sign_message(td::MutableSlice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return sign_message(signature.ubegin(), message.ubegin(), message.size()); - } - // used for ECDH (encrypt / decrypt) - bool compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub); - bool compute_shared_secret(td::MutableSlice secret, const PublicKey &Pub) { - CHECK(secret.size() == shared_secret_bytes); - return compute_shared_secret(secret.ubegin(), Pub); - } - // used for EC asymmetric decryption - bool compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]); - - const PublicKey &get_public_key() const { - return PubKey; - } - - private: - bool inited; - unsigned char privkey[privkey_bytes]; - unsigned char priv_salt[32]; - arith::Bignum priv_exp; - PublicKey PubKey; - - bool process_private_key(); - PrivateKey(const PrivateKey &) { - throw priv_key_no_copy(); - } - PrivateKey &operator=(const PrivateKey &) { - throw priv_key_no_copy(); - } -}; - -// use one TempKeyGenerator object a lot of times -class TempKeyGenerator { - enum { salt_size = 64 }; - unsigned char random_salt[salt_size]; - unsigned char buffer[privkey_bytes]; - - public: - TempKeyGenerator() { - prng::rand_gen().strong_rand_bytes(random_salt, salt_size); - } - ~TempKeyGenerator() { - memset(random_salt, 0, salt_size); - memset(buffer, 0, privkey_bytes); - } - - unsigned char *get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); // rand may be 0 - void create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); - - // sets temp_pub_key and shared_secret for one-time asymmetric encryption of message - bool create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], unsigned char secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); -}; - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Fp25519.cpp b/crypto/ellcurve/Fp25519.cpp deleted file mode 100644 index 7b248f64a..000000000 --- a/crypto/ellcurve/Fp25519.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -const Bignum& P25519() { - static const Bignum P25519 = (Bignum(1) << 255) - 19; - return P25519; -} - -td::Ref Fp25519() { - static const td::Ref Fp25519(true, P25519()); - return Fp25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.cpp b/crypto/ellcurve/Montgomery.cpp deleted file mode 100644 index ed71910d5..000000000 --- a/crypto/ellcurve/Montgomery.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Montgomery.h" - -#include -#include - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve; - -void MontgomeryCurve::init() { - assert(!((a_short + 2) & 3) && a_short >= 0); -} - -void MontgomeryCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - } - assert(!power_gen_xz(1).is_infty()); - assert(power_gen_xz(Order).is_infty()); -} - -// computes u(P+Q)*u(P-Q) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::add_xz(const MontgomeryCurve::PointXZ& P, - const MontgomeryCurve::PointXZ& Q) const { - Residue u = (P.X + P.Z) * (Q.X - Q.Z); - Residue v = (P.X - P.Z) * (Q.X + Q.Z); - return MontgomeryCurve::PointXZ(sqr(u + v), sqr(u - v)); -} - -// computes u(2P) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::double_xz(const MontgomeryCurve::PointXZ& P) const { - Residue u = sqr(P.X + P.Z); - Residue v = sqr(P.X - P.Z); - Residue w = u - v; - return PointXZ(u * v, w * (v + Residue(a_short, ring) * w)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_gen_xz(const Bignum& n) const { - return power_xz(Gu, n); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const Residue& u, const Bignum& n) const { - return power_xz(PointXZ(u), n); -} - -// computes u([n]P) in form X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const PointXZ& A, const Bignum& n) const { - assert(n >= 0); - if (n == 0) { - return PointXZ(ring); - } - - int k = n.num_bits(); - PointXZ P(A); - PointXZ Q(double_xz(P)); - for (int i = k - 2; i >= 0; --i) { - PointXZ PQ(add_xz(P, Q)); - PQ.X *= A.Z; - PQ.Z *= A.X; - if (n[i]) { - P = PQ; - Q = double_xz(Q); - } else { - Q = PQ; - P = double_xz(P); - } - } - return P; -} - -bool MontgomeryCurve::PointXZ::export_point_y(unsigned char buffer[32]) const { - if ((X + Z).is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_y().extract().export_lsb(buffer, 32); - return true; - } -} - -bool MontgomeryCurve::PointXZ::export_point_u(unsigned char buffer[32]) const { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_u().extract().export_lsb(buffer, 32); - return true; - } -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_u(const unsigned char point[32]) const { - Bignum u; - u.import_lsb(point, 32); - u[255] = 0; - return PointXZ(Residue(u, ring)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_y(const unsigned char point[32]) const { - Bignum y; - y.import_lsb(point, 32); - y[255] = 0; - return PointXZ(Residue(y, ring), true); -} - -const MontgomeryCurve& Curve25519() { - static const MontgomeryCurve Curve25519 = [] { - MontgomeryCurve res(486662, 9, Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - return res; - }(); - return Curve25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.h b/crypto/ellcurve/Montgomery.h deleted file mode 100644 index 94d852c06..000000000 --- a/crypto/ellcurve/Montgomery.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include - -#include "openssl/bignum.h" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve { - td::Ref ring; - int A_short; // v^2 = u^2 + Au + 1 - int Gu_short; // u(G) - int a_short; // (A+2)/4 - Residue A_; - Residue Gu; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - - void init(); - - public: - MontgomeryCurve(int _A, int _Gu, td::Ref _R) - : ring(_R) - , A_short(_A) - , Gu_short(_Gu) - , a_short((_A + 2) / 4) - , A_(_A, _R) - , Gu(_Gu, _R) - , P_(_R->get_modulus()) - , cofactor_short(0) { - init(); - } - - const Residue& get_gen_u() const { - return Gu; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - - void set_order_cofactor(const Bignum& order, int cof); - - struct PointXZ { - Residue X, Z; - PointXZ(Residue x, Residue z) : X(x), Z(z) { - x.same_ring(z); - } - PointXZ(td::Ref r) : X(r->one()), Z(r->zero()) { - } - explicit PointXZ(Residue u) : X(u), Z(u.ring_of().one()) { - } - explicit PointXZ(Residue y, bool) : X(y.ring_of().one() + y), Z(y.ring_of().one() - y) { - } - PointXZ(const PointXZ& P) : X(P.X), Z(P.Z) { - } - PointXZ& operator=(const PointXZ& P) { - X = P.X; - Z = P.Z; - return *this; - } - Residue get_u() const { - return X * inverse(Z); - } - Residue get_v(bool sign_v = false) const; - bool is_infty() const { - return Z.is_zero(); - } - Residue get_y() const { - return (X - Z) * inverse(X + Z); - } - bool export_point_y(unsigned char buffer[32]) const; - bool export_point_u(unsigned char buffer[32]) const; - void zeroize() { - X = Z = Z.ring_of().zero(); - } - }; - - PointXZ power_gen_xz(const Bignum& n) const; - PointXZ power_xz(const Residue& u, const Bignum& n) const; - PointXZ power_xz(const PointXZ& P, const Bignum& n) const; - PointXZ add_xz(const PointXZ& P, const PointXZ& Q) const; - PointXZ double_xz(const PointXZ& P) const; - - PointXZ import_point_u(const unsigned char point[32]) const; - PointXZ import_point_y(const unsigned char point[32]) const; -}; - -const MontgomeryCurve& Curve25519(); - -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.cpp b/crypto/ellcurve/TwEdwards.cpp deleted file mode 100644 index eb131f702..000000000 --- a/crypto/ellcurve/TwEdwards.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/TwEdwards.h" -#include -#include - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve; - -TwEdwardsCurve::TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R) - : ring(_R) - , D(_D) - , D2(_D + _D) - , Gy(_Gy) - , P_(_R->get_modulus()) - , cofactor_short(0) - , G(_R) - , O(_R) - , table_lines(0) - , table() { - init(); -} - -TwEdwardsCurve::~TwEdwardsCurve() { -} - -void TwEdwardsCurve::init() { - assert(D != ring->zero() && D != ring->convert(-1)); - O.X = O.Z = ring->one(); - G = SegrePoint(*this, Gy, 0); - assert(!G.XY.is_zero()); -} - -void TwEdwardsCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - assert(!power_gen(1).is_zero()); - assert(power_gen(L).is_zero()); - } -} - -TwEdwardsCurve::SegrePoint::SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign) - : XY(y), X(E.get_base_ring()), Y(y), Z(E.get_base_ring()->one()) { - Residue x(y.ring_ref()); - if (E.recover_x(x, y, x_sign)) { - XY *= x; - X = x; - } else { - XY = Y = Z = E.get_base_ring()->zero(); - } -} - -bool TwEdwardsCurve::recover_x(Residue& x, const Residue& y, bool x_sign) const { - // recovers x from equation -x^2+y^2 = 1+d*x^2*y^2 - Residue z = inverse(ring->one() + D * sqr(y)); - if (z.is_zero()) { - return false; - } - z *= sqr(y) - ring->one(); - Residue t = sqrt(z); - if (sqr(t) == z) { - x = (t.extract().odd() == x_sign) ? t : -t; - //std::cout << "x=" << x << ", y=" << y << std::endl; - return true; - } else { - return false; - } -} - -void TwEdwardsCurve::add_points(SegrePoint& Res, const SegrePoint& P, const SegrePoint& Q) const { - Residue a((P.X + P.Y) * (Q.X + Q.Y)); - Residue b((P.X - P.Y) * (Q.X - Q.Y)); - Residue c(P.Z * Q.Z * ring->convert(2)); - Residue d(P.XY * Q.XY * D2); - Residue x_num(a - b); // 2(x1y2+x2y1) - Residue y_num(a + b); // 2(x1x2+y1y2) - Residue x_den(c + d); // 2(1+dx1x2y1y2) - Residue y_den(c - d); // 2(1-dx1x2y1y2) - Res.X = x_num * y_den; // x = x_num/x_den, y = y_num/y_den - Res.Y = y_num * x_den; - Res.XY = x_num * y_num; - Res.Z = x_den * y_den; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::add_points(const SegrePoint& P, const SegrePoint& Q) const { - SegrePoint Res(ring); - add_points(Res, P, Q); - return Res; -} - -void TwEdwardsCurve::double_point(SegrePoint& Res, const SegrePoint& P) const { - add_points(Res, P, P); -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::double_point(const SegrePoint& P) const { - SegrePoint Res(ring); - double_point(Res, P); - return Res; -} - -// computes u([n]P) in form (xy,x,y,1)*Z -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_point(const SegrePoint& A, const Bignum& n, bool uniform) const { - assert(n >= 0); - if (n == 0) { - return O; - } - - int k = n.num_bits(); - SegrePoint P(A); - - if (uniform) { - SegrePoint Q(double_point(A)); - - for (int i = k - 2; i >= 0; --i) { - if (n[i]) { - add_points(P, P, Q); - double_point(Q, Q); - } else { - // we do more operations than necessary for uniformicity - add_points(Q, P, Q); - double_point(P, P); - } - } - } else { - for (int i = k - 2; i >= 0; --i) { - double_point(P, P); - if (n[i]) { - add_points(P, P, A); // may optimize further if A.z = 1 - } - } - } - return P; -} - -int TwEdwardsCurve::build_table() { - if (table.size()) { - return -1; - } - table_lines = (P_.num_bits() >> 2) + 2; - table.reserve(table_lines * 15 + 1); - table.emplace_back(get_base_point()); - for (int i = 0; i < table_lines; i++) { - for (int j = 0; j < 15; j++) { - table.emplace_back(add_points(table[15 * i + j], table[15 * i])); - } - } - return 1; -} - -int get_nibble(const Bignum& n, int idx) { - return n[idx * 4 + 3] * 8 + n[idx * 4 + 2] * 4 + n[idx * 4 + 1] * 2 + n[idx * 4]; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_gen(const Bignum& n, bool uniform) const { - if (uniform || n.num_bits() > table_lines * 4) { - return power_point(G, n, uniform); - } else if (n.is_zero()) { - return O; - } else { - int k = (n.num_bits() + 3) >> 2; - assert(k > 0 && k <= table_lines); - int x = get_nibble(n, k - 1); - assert(x > 0 && x < 16); - SegrePoint P(table[15 * (k - 1) + x - 1]); - for (int i = k - 2; i >= 0; i--) { - x = get_nibble(n, i); - assert(x >= 0 && x < 16); - if (x > 0) { - add_points(P, P, table[15 * i + x - 1]); - } - } - return P; - } -} - -bool TwEdwardsCurve::SegrePoint::export_point(unsigned char buffer[32], bool need_x) const { - if (!is_normalized()) { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z)); - Bignum y((Y * f).extract()); - assert(!y[255]); - if (need_x) { - y[255] = (X * f).extract().odd(); - } - y.export_lsb(buffer, 32); - } else { - Bignum y(Y.extract()); - assert(!y[255]); - if (need_x) { - y[255] = X.extract().odd(); - } - y.export_lsb(buffer, 32); - } - return true; -} - -bool TwEdwardsCurve::SegrePoint::export_point_u(unsigned char buffer[32]) const { - if (Z == Y) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z - Y)); - ((Z + Y) * f).extract().export_lsb(buffer, 32); - assert(!(buffer[31] & 0x80)); - return true; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::import_point(const unsigned char point[32], bool& ok) const { - Bignum y; - y.import_lsb(point, 32); - bool x_sign = y[255]; - y[255] = 0; - Residue yr(y, ring); - Residue xr(ring); - ok = recover_x(xr, yr, x_sign); - return ok ? SegrePoint(xr, yr) : SegrePoint(ring); -} - -const TwEdwardsCurve& Ed25519() { - static const TwEdwardsCurve Ed25519 = [] { - TwEdwardsCurve res(Fp25519()->frac(-121665, 121666), Fp25519()->frac(4, 5), Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - res.build_table(); - return res; - }(); - return Ed25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.h b/crypto/ellcurve/TwEdwards.h deleted file mode 100644 index c720ecca9..000000000 --- a/crypto/ellcurve/TwEdwards.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include "common/refcnt.hpp" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve { - public: - struct SegrePoint { - Residue XY, X, Y, Z; // if x=X/Z and y=Y/T, stores (xy,x,y,1)*Z*T - SegrePoint(td::Ref R) : XY(R), X(R), Y(R), Z(R) { - } - SegrePoint(const Residue& x, const Residue& y) : XY(x * y), X(x), Y(y), Z(y.ring_of().one()) { - } - SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign); - SegrePoint(const SegrePoint& P) : XY(P.XY), X(P.X), Y(P.Y), Z(P.Z) { - } - SegrePoint& operator=(const SegrePoint& P) { - XY = P.XY; - X = P.X; - Y = P.Y; - Z = P.Z; - return *this; - } - bool is_zero() const { - return X.is_zero() && (Y == Z); - } - bool is_valid() const { - return (XY * Z == X * Y) && !(XY.is_zero() && X.is_zero() && Y.is_zero() && Z.is_zero()); - } - bool is_finite() const { - return !Z.is_zero(); - } - bool is_normalized() const { - return Z == Z.ring_of().one(); - } - SegrePoint& normalize() { - auto f = inverse(Z); - XY *= f; - X *= f; - Y *= f; - Z = Z.ring_of().one(); - return *this; - } - SegrePoint& zeroize() { - XY = X = Y = Z = Z.ring_of().zero(); - return *this; - } - bool export_point(unsigned char buffer[32], bool need_x = true) const; - bool export_point_y(unsigned char buffer[32]) const { - return export_point(buffer, false); - } - bool export_point_u(unsigned char buffer[32]) const; - Residue get_y() const { - return Y * inverse(Z); - } - Residue get_x() const { - return X * inverse(Z); - } - Residue get_u() const { - return (Z + Y) * inverse(Z - Y); - } - void negate() { - XY.negate(); - X.negate(); - } - }; - - private: - td::Ref ring; - Residue D; - Residue D2; - Residue Gy; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - SegrePoint G; - SegrePoint O; - int table_lines; - std::vector table; - - void init(); - - public: - TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R); - ~TwEdwardsCurve(); - const Residue& get_gen_y() const { - return Gy; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - const SegrePoint& get_base_point() const { - return G; - } - - void set_order_cofactor(const Bignum& order, int cof); - bool recover_x(Residue& x, const Residue& y, bool x_sign) const; - - void add_points(SegrePoint& R, const SegrePoint& P, const SegrePoint& Q) const; - SegrePoint add_points(const SegrePoint& P, const SegrePoint& Q) const; - void double_point(SegrePoint& R, const SegrePoint& P) const; - SegrePoint double_point(const SegrePoint& P) const; - SegrePoint power_point(const SegrePoint& A, const Bignum& n, bool uniform = false) const; - SegrePoint power_gen(const Bignum& n, bool uniform = false) const; - int build_table(); - - SegrePoint import_point(const unsigned char point[32], bool& ok) const; -}; - -std::ostream& operator<<(std::ostream& os, const TwEdwardsCurve::SegrePoint& P); -const TwEdwardsCurve& Ed25519(); -} // namespace ellcurve diff --git a/crypto/ellcurve/p256.cpp b/crypto/ellcurve/p256.cpp index de5393723..684ac8ca0 100644 --- a/crypto/ellcurve/p256.cpp +++ b/crypto/ellcurve/p256.cpp @@ -58,7 +58,7 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice SCOPE_EXIT { EVP_MD_CTX_free(md_ctx); }; - if (EVP_DigestVerifyInit(md_ctx, nullptr, nullptr, nullptr, pkey) <= 0) { + if (EVP_DigestVerifyInit(md_ctx, nullptr, EVP_sha256(), nullptr, pkey) <= 0) { return td::Status::Error("Can't init DigestVerify"); } ECDSA_SIG* sig = ECDSA_SIG_new(); @@ -71,7 +71,10 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice BIGNUM* r = BN_bin2bn(buf, 33, nullptr); std::copy(signature.ubegin() + 32, signature.ubegin() + 64, buf + 1); BIGNUM* s = BN_bin2bn(buf, 33, nullptr); + CHECK(r != nullptr && s != nullptr); if (ECDSA_SIG_set0(sig, r, s) != 1) { + BN_free(r); + BN_free(s); return td::Status::Error("Invalid signature"); } unsigned char* signature_encoded = nullptr; diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 976093f80..a4b1e144d 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -2,7 +2,7 @@ library TVM_Asm // simple TVM Assembler namespace Asm Asm definitions -"0.4.5" constant asm-fif-version +"0.4.6" constant asm-fif-version variable @atend variable @was-split @@ -697,6 +697,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME +x{CF50} @Defop BTOS { tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= @@ -750,26 +751,37 @@ x{D723} @Defop SDSKIPLAST x{D724} @Defop SDSUBSTR x{D726} @Defop SDBEGINSX x{D727} @Defop SDBEGINSXQ + +/* +if (rembits >= opcodebits) or (rembits <= 18) + just write +else + PUSHSLICE SDBEGINSXQ + +--- +18 - size of PUSHREFSLICE +*/ + { tuck sbits tuck 5 + 3 >> swap x{D72A_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINS:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits ((sbits + 5) // 8) * 8 + 3 + 21 + over brembits <= // total op bits <= remaining free bits in slice + { > swap x{D72E_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINSQ:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits + over brembits <= // total op bits <= remaining free bits in slice + { ()); auto tmp_vars = x->pre_compile(code); code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); - code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + code.emplace_back(loc, Op::_Nop); // This is necessary to prevent SIGSEGV! // It is REQUIRED to execute "optimizations" as in func.cpp code.simplify_var_types(); code.prune_unreachable_code(); diff --git a/crypto/openssl/digest.h b/crypto/openssl/digest.h deleted file mode 100644 index d0dfc4c08..000000000 --- a/crypto/openssl/digest.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once -#include - -#include -#include - -#include "td/utils/Slice.h" - -namespace digest { -struct OpensslEVP_SHA1 { - enum { digest_bytes = 20 }; - static const EVP_MD *get_evp() { - return EVP_sha1(); - } -}; - -struct OpensslEVP_SHA256 { - enum { digest_bytes = 32 }; - static const EVP_MD *get_evp() { - return EVP_sha256(); - } -}; - -struct OpensslEVP_SHA512 { - enum { digest_bytes = 64 }; - static const EVP_MD *get_evp() { - return EVP_sha512(); - } -}; - -template -class HashCtx { - EVP_MD_CTX *ctx{nullptr}; - void init(); - void clear(); - - public: - enum { digest_bytes = H::digest_bytes }; - HashCtx() { - init(); - } - HashCtx(const void *data, std::size_t len) { - init(); - feed(data, len); - } - ~HashCtx() { - clear(); - } - void reset(); - void feed(const void *data, std::size_t len); - void feed(td::Slice slice) { - feed(slice.data(), slice.size()); - } - std::size_t extract(unsigned char buffer[digest_bytes]); - std::size_t extract(td::MutableSlice slice); - std::string extract(); -}; - -template -void HashCtx::init() { - ctx = EVP_MD_CTX_create(); - reset(); -} - -template -void HashCtx::reset() { - EVP_DigestInit_ex(ctx, H::get_evp(), 0); -} - -template -void HashCtx::clear() { - EVP_MD_CTX_destroy(ctx); - ctx = nullptr; -} - -template -void HashCtx::feed(const void *data, std::size_t len) { - EVP_DigestUpdate(ctx, data, len); -} - -template -std::size_t HashCtx::extract(unsigned char buffer[digest_bytes]) { - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return olen; -} - -template -std::size_t HashCtx::extract(td::MutableSlice slice) { - return extract(slice.ubegin()); -} - -template -std::string HashCtx::extract() { - unsigned char buffer[digest_bytes]; - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return std::string((char *)buffer, olen); -} - -typedef HashCtx SHA1; -typedef HashCtx SHA256; -typedef HashCtx SHA512; - -template -std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(buffer); -} - -template -std::size_t hash_two_str(unsigned char buffer[T::digest_bytes], const void *data1, std::size_t size1, const void *data2, - std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(buffer); -} - -template -std::string hash_str(const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(); -} - -template -std::string hash_two_str(const void *data1, std::size_t size1, const void *data2, std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(); -} -} // namespace digest diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 2adeef1d7..3e36f7e99 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -21,6 +21,7 @@ #include #include +#include #include "td/utils/Slice.h" @@ -124,9 +125,63 @@ std::string HashCtx::extract() { } typedef HashCtx SHA1; -typedef HashCtx SHA256; typedef HashCtx SHA512; +struct SHA256Tag {}; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +template <> +struct HashCtx { + public: + enum { digest_bytes = 32 }; + + HashCtx() { + SHA256_Init(&ctx_); + } + + HashCtx(const void *data, std::size_t len) : HashCtx() { + feed(data, len); + } + + void reset() { + SHA256_Init(&ctx_); + } + + void feed(const void *data, std::size_t len) { + SHA256_Update(&ctx_, data, len); + } + + void feed(td::Slice slice) { + feed(slice.data(), slice.size()); + } + + std::size_t extract(unsigned char buffer[digest_bytes]) { + SHA256_Final(buffer, &ctx_); + return digest_bytes; + } + + std::size_t extract(td::MutableSlice slice) { + CHECK(slice.size() == digest_bytes); + SHA256_Final(slice.ubegin(), &ctx_); + return digest_bytes; + } + + std::string extract() { + unsigned char buffer[digest_bytes]; + extract(buffer); + return std::string((char *)buffer, digest_bytes); + } + + private: + SHA256_CTX ctx_; +}; + +#pragma GCC diagnostic pop + +typedef HashCtx SHA256; + template std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { T hasher(data, size); diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index de21cd47c..25873eefe 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -49,6 +49,9 @@ variable @default-subwallet-id 8 constant capReportVersion 16 constant capSplitMergeTransactions 32 constant capShortDequeue +64 constant capStoreOutMsgQueueSize +128 constant capMsgMetadata +256 constant capDeferMessages // max-validators masterchain-validators min-validators -- { swap rot 16 config! } : config.validator_num! @@ -176,8 +179,16 @@ variable special-dict // bytes-limits gas-limits lt-limits -- c { } : make-block-limits +// bytes-limits gas-limits lt-limits collated-data-limits imported_queue_bytes imported_queue_msgs -- c +{ +} : make-block-limits-v2 + { make-block-limits 22 config! } : config.mc_block_limits! { make-block-limits 23 config! } : config.block_limits! +{ make-block-limits-v2 22 config! } : config.mc_block_limits_v2! +{ make-block-limits-v2 23 config! } : config.block_limits_v2! // mc-block-create-fee bc-block-create-fee { } : make-block-create-fees diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 82757c22b..7d21dd7d9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,210 +1,478 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.9 +tolk 1.0 + +/// In Tolk v1.x there would be a type `map`. +/// Currently, working with dictionaries is still low-level, with raw cells. +/// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. +/// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". +type dict = cell? /** - Tuple manipulation primitives. - Elements of a tuple can be of arbitrary type. - Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. + Tuple manipulation primitives. + Elements of a tuple can be of arbitrary type. + Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. */ /// Creates a tuple with zero elements. @pure fun createEmptyTuple(): tuple - asm "NIL"; + asm "NIL" /// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. /// If its size exceeds 255, throws a type check exception. @pure -fun tuplePush(mutate self: tuple, value: T): void - asm "TPUSH"; +fun tuple.push(mutate self, value: T): void + asm "TPUSH" /// Returns the first element of a non-empty tuple. -/// `t.0` is actually the same as `t.tupleFirst()` +/// `t.0` is actually the same as `t.first()` @pure -fun tupleFirst(self: tuple): T - asm "FIRST"; +fun tuple.first(self): T + asm "FIRST" /// Returns the [`index`]-th element of a tuple. -/// `t.i` is actually the same as `t.tupleAt(i)` +/// `t.i` is actually the same as `t.get(i)` @pure -fun tupleAt(self: tuple, index: int): T - builtin; +fun tuple.get(self, index: int): T + builtin /// Sets the [`index`]-th element of a tuple to a specified value /// (element with this index must already exist, a new element isn't created). -/// `t.i = value` is actually the same as `t.tupleSetAt(value, i)` +/// `t.i = value` is actually the same as `t.set(value, i)` @pure -fun tupleSetAt(mutate self: tuple, value: T, index: int): void - builtin; +fun tuple.set(mutate self, value: T, index: int): void + builtin /// Returns the size of a tuple (elements count in it). @pure -fun tupleSize(self: tuple): int - asm "TLEN"; +fun tuple.size(self): int + asm "TLEN" /// Returns the last element of a non-empty tuple. @pure -fun tupleLast(self: tuple): T - asm "LAST"; +fun tuple.last(self): T + asm "LAST" + +/// Pops and returns the last element of a non-empty tuple. +@pure +fun tuple.pop(mutate self): T + asm "TPOP" /** Mathematical primitives. */ +/// Converts a constant floating-point string to nanotoncoins. +/// Example: `ton("0.05")` is equal to 50000000. +/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error +@pure +fun ton(floatString: slice): coins + builtin + /// Computes the minimum of two integers. @pure fun min(x: int, y: int): int - asm "MIN"; + asm "MIN" /// Computes the maximum of two integers. @pure fun max(x: int, y: int): int - asm "MAX"; + asm "MAX" /// Sorts two integers. +/// Example: `minMax(x, y)` with (x=20, y=10) and with (x=10, y=20) returns (10, 20) @pure fun minMax(x: int, y: int): (int, int) - asm "MINMAX"; + asm "MINMAX" /// Computes the absolute value of an integer. @pure fun abs(x: int): int - asm "ABS"; + asm "ABS" /// Returns the sign of an integer: `-1` if x < 0, `0` if x == 0, `1` if x > 0. @pure fun sign(x: int): int - asm "SGN"; + asm "SGN" /// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) @pure fun divMod(x: int, y: int): (int, int) - asm "DIVMOD"; + asm "DIVMOD" /// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) @pure fun modDiv(x: int, y: int): (int, int) - asm(-> 1 0) "DIVMOD"; + asm(-> 1 0) "DIVMOD" /// Computes multiple-then-divide: floor([x] * [y] / [z]). /// The intermediate result is stored in a 513-bit integer to prevent precision loss. @pure fun mulDivFloor(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). @pure fun mulDivRound(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). @pure fun mulDivCeil(x: int, y: int, z: int): int - builtin; + builtin /// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) @pure fun mulDivMod(x: int, y: int, z: int): (int, int) - builtin; + builtin /** - Global getters of environment and contract state. + Global getters and setters of current contract state. */ -const MASTERCHAIN = -1; -const BASECHAIN = 0; +/// `contract` is a built-in struct, it has only static methods. +/// Example: `contract.getCode()` and other methods. +struct contract -/// Returns current Unix timestamp (in seconds). +/// Returns the internal address of the current smart contract. +/// If necessary, it can be parsed further using [address.getWorkchain] and others. @pure -fun now(): int - asm "NOW"; - -/// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. -/// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. -@pure -fun getMyAddress(): slice - asm "MYADDR"; +fun contract.getAddress(): address + asm "MYADDR" /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. /// Note that RAW primitives such as [sendMessage] do not update this field. @pure -fun getMyOriginalBalance(): int - asm "BALANCE" "FIRST"; +fun contract.getOriginalBalance(): coins + asm "BALANCE" "FIRST" -/// Same as [getMyOriginalBalance], but returns a tuple: +/// Same as [contract.getOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; -/// `cell` — a dictionary with 32-bit keys representing the balance of "extra currencies". -@pure -fun getMyOriginalBalanceWithExtraCurrencies(): [int, cell?] - asm "BALANCE"; - -/// Returns the logical time of the current transaction. -@pure -fun getLogicalTime(): int - asm "LTIME"; - -/// Returns the starting logical time of the current block. -@pure -fun getCurrentBlockLogicalTime(): int - asm "BLOCKLT"; - -/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +/// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure -fun getBlockchainConfigParam(x: int): cell? - asm "CONFIGOPTPARAM"; +fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] + asm "BALANCE" /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @pure -fun getContractData(): cell - asm "c4 PUSH"; +fun contract.getData(): cell + asm "c4 PUSH" /// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. -fun setContractData(c: cell): void - asm "c4 POP"; +fun contract.setData(c: cell): void + asm "c4 POP" /// Retrieves code of smart-contract from c7 @pure -fun getContractCode(): cell - asm "MYCODE"; +fun contract.getCode(): cell + asm "MYCODE" /// Creates an output action that would change this smart contract code to that given by cell [newCode]. /// Notice that this change will take effect only after the successful termination of the current run of the smart contract. -fun setContractCodePostponed(newCode: cell): void - asm "SETCODE"; +fun contract.setCodePostponed(newCode: cell): void + asm "SETCODE" + + +/** + Global getters of current blockchain (environment) state. +*/ + +const MASTERCHAIN = -1 +const BASECHAIN = 0 + +/// `blockchain` is a built-in struct, it has only static methods. +/// Example: `blockchain.configParam(16)` and other methods. +struct blockchain + +/// Returns current Unix timestamp (in seconds). +@pure +fun blockchain.now(): int + asm "NOW" + +/// Returns the logical time of the current transaction. +@pure +fun blockchain.logicalTime(): int + asm "LTIME" + +/// Returns the starting logical time of the current block. +@pure +fun blockchain.currentBlockLogicalTime(): int + asm "BLOCKLT" + +/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +@pure +fun blockchain.configParam(x: int): cell? + asm "CONFIGOPTPARAM" /// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) /// so that the current execution is considered “successful” with the saved values even if an exception /// in Computation Phase is thrown later. fun commitContractDataAndActions(): void - asm "COMMIT"; + asm "COMMIT" + + +/** + Auto packing structures to/from cells. +*/ + +/// PackOptions allows you to control behavior of `obj.toCell()` and similar functions. +struct PackOptions { + /// when a struct has a field of type `bits128` and similar (it's a slice under the hood), + /// by default, compiler inserts runtime checks (get bits/refs count + compare with 128 + compare with 0); + /// these checks ensure that serialized binary data will be correct, but they cost gas; + /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources), + /// set this option to true to disable runtime checks; + /// note: `int32` and other are always validated for overflow without any extra gas, + /// so this flag controls only rarely used `bitsN` type + skipBitsNValidation: bool = false, +} + +/// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. +struct UnpackOptions { + /// after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; + /// it's the default behavior, it ensures that you've fully described data you're reading with a struct; + /// example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; + /// note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); + /// note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny]; + /// note: `lazy` ignores this option, because it reads fields on demand or even skips them + assertEndAfterReading: bool = true, + + /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; + /// similarly, for a union type, this is thrown when none of the opcodes match; + /// note: `lazy` ignores this option if you have `else` in `match` (you write custom logic there) + throwIfOpcodeDoesNotMatch: int = 63, +} + +/// Convert anything to a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st: MyStorage = { ... }; +/// contract.setData(st.toCell()); +/// ``` +/// Internally, a builder is created, all fields are serialized one by one, and a builder is flushed +/// (beginCell() + serialize fields + endCell()). +@pure +fun T.toCell(self, options: PackOptions = {}): Cell + builtin + +/// Parse anything from a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st = MyStorage.fromCell(contract.getData()); +/// ``` +/// Internally, a cell is unpacked to a slice, and that slice is parsed +/// (packedCell.beginParse() + read from slice). +@pure +fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T + builtin + +/// Parse anything from a slice (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var msg = CounterIncrement.fromSlice(cs); +/// ``` +/// All fields are read from a slice immediately. +/// If a slice is corrupted, an exception is thrown (most likely, excode 9 "cell underflow"). +/// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. +/// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. +@pure +fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T + builtin + +/// Parse anything from a slice, shifting its internal pointer. +/// Similar to `slice.loadUint()` and others, but allows loading structures. +/// Example: +/// ``` +/// var st: MyStorage = cs.loadAny(); // or cs.loadAny() +/// ``` +/// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. +/// Note: [options.assertEndAfterReading] is ignored by this function, because it's actually intended +/// to read data from the middle. +@pure +fun slice.loadAny(mutate self, options: UnpackOptions = {}): T + builtin + +/// Skip anything in a slice, shifting its internal pointer. +/// Similar to `slice.skipBits()` and others, but allows skipping structures. +/// Example: +/// ``` +/// struct TwoInts { a: int32; b: int32; } +/// cs.skipAny(); // skips 64 bits +/// ``` +@pure +fun slice.skipAny(mutate self, options: UnpackOptions = {}): self + builtin + +/// Store anything to a builder. +/// Similar to `builder.storeUint()` and others, but allows storing structures. +/// Example: +/// ``` +/// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); +/// ``` +@pure +fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self + builtin + +/// Returns serialization prefix of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `240`. +@pure +fun T.getDeclaredPackPrefix(): int + builtin + +/// Returns serialization prefix length of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `16`. +@pure +fun T.getDeclaredPackPrefixLen(): int + builtin + +/// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). +/// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), +/// this method can help you overcome this, if you really need to check input consistency. +/// Example: +/// ``` +/// val msg = lazy CounterMessage.fromSlice(s); +/// match (msg) { // it's a lazy match, without creating a union on the stack +/// CounterIncrement => { +/// ... +/// newCounter = curCounter + msg.incBy; // `incBy` loaded here, on demand +/// msg.forceLoadLazyObject().assertEnd() // the purpose: get remainder +/// } +/// } +/// ``` +/// Note: while [slice.assertEnd] may seem reasonable, these checks are avoided in practice, +/// because the purpose of `lazy` is to auto-detect and load only necessary fields, not up to the end. +@pure +fun T.forceLoadLazyObject(self): slice + builtin + +/// Cell represents a typed cell reference (as opposed to untyped `cell`). +/// Example: +/// ``` +/// struct ExtraData { ... } +/// +/// struct MyStorage { +/// ... +/// extra: Cell; // TL-B `^ExtraData` +/// optional: Cell?; // TL-B `(Maybe ^ExtraData)` +/// code: cell; // TL-B `^Cell` +/// data: cell?; // TL-B `(Maybe ^Cell)` +/// } +/// ``` +/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; +/// you should manually call `st.extra.load()` to get T (ExtraData in this example). +struct Cell { + tvmCell: cell +} + +/// Parse data from already loaded cell reference. +/// Example: +/// ``` +/// struct MyStorage { ... extra: Cell; } +/// +/// var st = MyStorage.fromCell(contract.getData()); +/// // st.extra is cell; if we need to unpack it, we do +/// var extra = st.extra.load(); // it's ExtraData, unpacked from loaded ref +/// ``` +@pure +fun Cell.load(self, options: UnpackOptions = {}): T + builtin + +/// Converts a typed cell into a slice. +@pure +fun Cell.beginParse(self): slice + asm "CTOS" + +/// Returns hash of a typed cell, same as [cell.hash]. +@pure +fun Cell.hash(self): uint256 + asm "HASHCU" + +/// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. +/// Example: +/// ``` +/// struct JettonMessage { +/// ... some fields +/// forwardPayload: RemainingBitsAndRefs; +/// } +/// ``` +/// When you deserialize JettonMessage, forwardPayload contains "everything left after reading fields above". +type RemainingBitsAndRefs = slice + +/// Creates a cell with zero bits and references. +/// Equivalent to `beginCell().endCell()` but cheaper. +@pure +fun createEmptyCell(): cell + asm " PUSHREF" + +/// Creates a slice with zero remaining bits and references. +/// Equivalent to `beginCell().endCell().beginParse()` but cheaper. +@pure +fun createEmptySlice(): slice + asm "x{} PUSHSLICE" /** - Signature checks, hashing, cryptography. + Signature checks, hashing, cryptography. */ -/// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +/// Compile-time function that calculates crc32 of a constant string. +/// Example: `const op = stringCrc32("some_str")` = 4013618352 = 0xEF3AF4B0 +/// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc32(constString: slice): int + builtin + +/// Compile-time function that calculates crc16 (XMODEM) of a constant string. +/// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F +/// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc16(constString: slice): int + builtin + +/// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. +/// Example: `const hash = stringSha256("some_crypto_key")` +/// Note: it's a compile-time function, `stringSha256(slice_var)` does not work. +/// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. +@pure +fun stringSha256(constString: slice): int + builtin + +/// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. +/// Example: `const minihash = stringSha256_32("some_crypto_key")` +/// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringSha256_32(constString: slice): int + builtin + +/// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. +/// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) +/// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringToBase256(constString: slice): int + builtin + +/// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure -fun cellHash(c: cell): int - asm "HASHCU"; +fun cell.hash(self): uint256 + asm "HASHCU" -/// Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +/// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created -/// and its hash computed by [cellHash]. +/// and its hash computed by [cell.hash]. @pure -fun sliceHash(s: slice): int - asm "HASHSU"; +fun slice.hash(self): uint256 + asm "HASHSU" -/// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +/// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun stringHash(s: slice): int - asm "SHA256U"; +fun slice.bitsHash(self): uint256 + asm "SHA256U" /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) /// using [publicKey] (also represented by a 256-bit unsigned integer). @@ -215,7 +483,7 @@ fun stringHash(s: slice): int /// the second hashing occurring inside `CHKSIGNS`. @pure fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool - asm "CHKSIGNU"; + asm "CHKSIGNU" /// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, /// similarly to [isSignatureValid]. @@ -224,38 +492,45 @@ fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool /// with sha256 used to reduce [data] to the 256-bit number that is actually signed. @pure fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool - asm "CHKSIGNS"; + asm "CHKSIGNS" + +/// `random` is a built-in struct, it has only static methods. +/// Example: `random.uint256()` and other methods. +struct random /// Generates a new pseudo-random unsigned 256-bit integer x. -fun random(): int - asm "RANDU256"; +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.uint256(): uint256 + asm "RANDU256" -/// Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). -/// More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. -fun randomRange(range: int): int - asm "RAND"; +/// Generates a new pseudo-random integer z in the range 0..limit−1 (or limit..−1, if upto < 0). +/// More precisely, an unsigned random value x is generated as in random; then z := x * limit / 2^256 is computed. +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.range(limit: int): int + asm "RAND" -/// Returns the current random seed as an unsigned 256-bit integer. +/// Returns the current random seed used to generate pseudo-random numbers. @pure -fun randomGetSeed(): int - asm "RANDSEED"; +fun random.getSeed(): uint256 + asm "RANDSEED" -/// Sets the random seed to unsigned 256-bit seed. -fun randomSetSeed(seed: int): void - asm "SETRAND"; +/// Sets the random seed to the provided value. +fun random.setSeed(seed: uint256): void + asm "SETRAND" -/// Initializes (mixes) random seed with unsigned 256-bit integer x. -fun randomizeBy(x: int): void - asm "ADDRAND"; +/// Initializes (mixes) random seed with the provided value. +fun random.initializeBy(mixSeedWith: uint256): void + asm "ADDRAND" -/// Initializes random seed using current time. Don't forget to call this before calling `random`! -fun randomizeByLogicalTime(): void - asm "LTIME" "ADDRAND"; +/// Initializes random seed with current time to make random generation unpredictable. +/// Typically, you call this function once before calling [random.uint256] / [random.range]. +fun random.initialize(): void + asm "LTIME" "ADDRAND" /** - Size computation primitives. - They may be useful for computing storage fees of user-provided data. + Size computation primitives. + They may be useful for computing storage fees of user-provided data. */ /// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. @@ -268,489 +543,838 @@ fun randomizeByLogicalTime(): void /// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and /// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. @pure -fun calculateCellSize(c: cell, maxCells: int): (int, int, int, bool) - asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; +fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) + asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" -/// Similar to [calculateCellSize], but accepting a `slice` [s] instead of a `cell`. +/// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. /// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; /// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. @pure -fun calculateSliceSize(s: slice, maxCells: int): (int, int, int, bool) - asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; +fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) + asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" -/// A non-quiet version of [calculateCellSize] that throws a cell overflow exception (`8`) on failure. -fun calculateCellSizeStrict(c: cell, maxCells: int): (int, int, int) - asm "CDATASIZE"; +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) + asm "CDATASIZE" -/// A non-quiet version of [calculateSliceSize] that throws a cell overflow exception (`8`) on failure. -fun calculateSliceSizeStrict(s: slice, maxCells: int): (int, int, int) - asm "SDATASIZE"; +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) + asm "SDATASIZE" -/// Returns the depth of `cell` [c]. +/// Returns the depth of a `cell`. /// If [c] has no references, then return `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. /// If [c] is a `null` instead of a cell, returns zero. @pure -fun getCellDepth(c: cell?): int - asm "CDEPTH"; +fun cell?.depth(self): int + asm "CDEPTH" -/// Returns the depth of `slice` [s]. +/// Returns the depth of a `slice`. /// If [s] has no references, then returns `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. @pure -fun getSliceDepth(s: slice): int - asm "SDEPTH"; +fun slice.depth(self): int + asm "SDEPTH" -/// Returns the depth of `builder` [b]. +/// Returns the depth of a `builder`. /// If no cell references are stored in [b], then returns 0; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. @pure -fun getBuilderDepth(b: builder): int - asm "BDEPTH"; +fun builder.depth(self): int + asm "BDEPTH" + +/// Returns the number of stack slots anyVariable occupies (works at compile-time). +/// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. +/// Example: sizeof(somePoint) = 2 for `struct Point { x:int, y: int }`: two fields one slot per each. +/// Useful for debugging or when preparing stack contents for RUNVM. +@pure +fun sizeof(anyVariable: T): int + builtin /** - Debug primitives. - Only works for local TVM execution with debug level verbosity. + Debug primitives. + Only works for local TVM execution with debug level verbosity. */ +/// `debug` is a built-in struct, it has only static methods. +/// Example: `debug.print(v)` and other methods. +struct debug + /// Dump a variable [x] to the debug log. -fun debugPrint(x: T): void - builtin; +fun debug.print(x: T): void + builtin /// Dump a string [x] to the debug log. -fun debugPrintString(x: T): void - builtin; +fun debug.printString(x: T): void + builtin /// Dumps the stack (at most the top 255 values) and shows the total stack depth. -fun debugDumpStack(): void - builtin; +fun debug.dumpStack(): void + builtin /** - Slice primitives: parsing cells. - When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). - When you _preload_ some data, you just get the result without mutating the slice. + Slice primitives: parsing cells. + When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). + When you _preload_ some data, you just get the result without mutating the slice. */ -/// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +/// Compile-time function that converts a constant hex-encoded string to N/2 bytes. +/// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` +/// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringHexToSlice(constStringBytesHex: slice): slice + builtin + +/// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. @pure -fun beginParse(c: cell): slice - asm "CTOS"; +fun cell.beginParse(self): slice + asm "CTOS" -/// Checks if slice is empty. If not, throws an exception. -fun assertEndOfSlice(self: slice): void - asm "ENDS"; +/// Checks if slice is empty. If not, throws an exception with code 9. +fun slice.assertEnd(self): void + asm "ENDS" /// Loads the next reference from the slice. @pure -fun loadRef(mutate self: slice): cell - asm( -> 1 0) "LDREF"; +fun slice.loadRef(mutate self): cell + asm( -> 1 0) "LDREF" /// Preloads the next reference from the slice. @pure -fun preloadRef(self: slice): cell - asm "PLDREF"; +fun slice.preloadRef(self): cell + asm "PLDREF" /// Loads a signed [len]-bit integer from a slice. @pure -fun loadInt(mutate self: slice, len: int): int - builtin; +fun slice.loadInt(mutate self, len: int): int + builtin /// Loads an unsigned [len]-bit integer from a slice. @pure -fun loadUint(mutate self: slice, len: int): int - builtin; +fun slice.loadUint(mutate self, len: int): int + builtin /// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. @pure -fun loadBits(mutate self: slice, len: int): slice - builtin; +fun slice.loadBits(mutate self, len: int): slice + builtin /// Preloads a signed [len]-bit integer from a slice. @pure -fun preloadInt(self: slice, len: int): int - builtin; +fun slice.preloadInt(self, len: int): int + builtin /// Preloads an unsigned [len]-bit integer from a slice. @pure -fun preloadUint(self: slice, len: int): int - builtin; +fun slice.preloadUint(self, len: int): int + builtin /// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. @pure -fun preloadBits(self: slice, len: int): slice - builtin; +fun slice.preloadBits(self, len: int): slice + builtin /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure -fun loadCoins(mutate self: slice): int - asm( -> 1 0) "LDGRAMS"; +fun slice.loadCoins(mutate self): coins + asm( -> 1 0) "LDGRAMS" /// Loads bool (-1 or 0) from a slice @pure -fun loadBool(mutate self: slice): bool - asm( -> 1 0) "1 LDI"; +fun slice.loadBool(mutate self): bool + asm( -> 1 0) "1 LDI" /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure -fun skipBits(mutate self: slice, len: int): self - asm "SDSKIPFIRST"; +fun slice.skipBits(mutate self, len: int): self + builtin /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getFirstBits(self: slice, len: int): slice - asm "SDCUTFIRST"; +fun slice.getFirstBits(self, len: int): slice + asm "SDCUTFIRST" /// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun removeLastBits(mutate self: slice, len: int): self - asm "SDSKIPLAST"; +fun slice.removeLastBits(mutate self, len: int): self + asm "SDSKIPLAST" /// Returns the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getLastBits(self: slice, len: int): slice - asm "SDCUTLAST"; +fun slice.getLastBits(self, len: int): slice + asm "SDCUTLAST" + +/// Returns `0 ≤ len ≤ 1023` bits of a slice starting from `0 ≤ offset ≤ 1023`. +/// (in other words, extracts a bit substring `[offset, len)` out of the slice) +@pure +fun slice.getMiddleBits(self, offset: int, len: int): slice + asm "SDSUBSTR" /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure -fun loadDict(mutate self: slice): cell? - asm( -> 1 0) "LDDICT"; +fun slice.loadDict(mutate self): dict + asm( -> 1 0) "LDDICT" /// Preloads a dictionary (cell) from a slice. @pure -fun preloadDict(self: slice): cell? - asm "PLDDICT"; +fun slice.preloadDict(self): dict + asm "PLDDICT" -/// Loads a dictionary as [loadDict], but returns only the remainder of the slice. +/// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. @pure -fun skipDict(mutate self: slice): self - asm "SKIPDICT"; +fun slice.skipDict(mutate self): self + asm "SKIPDICT" /// Loads (Maybe ^Cell) from a slice. /// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. @pure -fun loadMaybeRef(mutate self: slice): cell? - asm( -> 1 0) "LDOPTREF"; +fun slice.loadMaybeRef(mutate self): cell? + asm( -> 1 0) "LDOPTREF" /// Preloads (Maybe ^Cell) from a slice. @pure -fun preloadMaybeRef(self: slice): cell? - asm "PLDOPTREF"; +fun slice.preloadMaybeRef(self): cell? + asm "PLDOPTREF" /// Loads (Maybe ^Cell), but returns only the remainder of the slice. @pure -fun skipMaybeRef(mutate self: slice): self - asm "SKIPOPTREF"; +fun slice.skipMaybeRef(mutate self): self + asm "SKIPOPTREF" /** - Builder primitives: constructing cells. - When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). - All the primitives below first check whether there is enough space in the `builder`, - and only then check the range of the value being serialized. + Builder primitives: constructing cells. + When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. */ /// Creates a new empty builder. @pure fun beginCell(): builder - asm "NEWC"; + asm "NEWC" /// Converts a builder into an ordinary `cell`. @pure -fun endCell(self: builder): cell - asm "ENDC"; +fun builder.endCell(self): cell + asm "ENDC" /// Stores a reference to a cell into a builder. @pure -fun storeRef(mutate self: builder, c: cell): self - asm(c self) "STREF"; +fun builder.storeRef(mutate self, c: cell): self + asm(c self) "STREF" /// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). @pure -fun storeInt(mutate self: builder, x: int, len: int): self - builtin; +fun builder.storeInt(mutate self, x: int, len: int): self + builtin /// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). @pure -fun storeUint(mutate self: builder, x: int, len: int): self - builtin; +fun builder.storeUint(mutate self, x: int, len: int): self + builtin /// Stores a slice into a builder. @pure -fun storeSlice(mutate self: builder, s: slice): self - asm "STSLICER"; +fun builder.storeSlice(mutate self, s: slice): self + asm(s self) "STSLICE" + +/// Stores an address into a builder. +@pure +fun builder.storeAddress(mutate self, addr: address): self + asm(addr self) "STSLICE" /// Stores amount of Toncoins into a builder. @pure -fun storeCoins(mutate self: builder, x: int): self - asm "STGRAMS"; +fun builder.storeCoins(mutate self, x: coins): self + builtin /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure -fun storeBool(mutate self: builder, x: bool): self - asm(x self) "1 STI"; +fun builder.storeBool(mutate self, x: bool): self + builtin /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure -fun storeDict(mutate self: builder, c: cell?): self - asm(c self) "STDICT"; +fun builder.storeDict(mutate self, c: dict): self + asm(c self) "STDICT" /// Stores (Maybe ^Cell) into a builder. /// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. @pure -fun storeMaybeRef(mutate self: builder, c: cell?): self - asm(c self) "STOPTREF"; +fun builder.storeMaybeRef(mutate self, c: cell?): self + asm(c self) "STOPTREF" /// Concatenates two builders. @pure -fun storeBuilder(mutate self: builder, from: builder): self - asm "STBR"; +fun builder.storeBuilder(mutate self, from: builder): self + asm(from self) "STB" /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure -fun storeAddressNone(mutate self: builder): self - asm "b{00} STSLICECONST"; +fun builder.storeAddressNone(mutate self): self + asm "b{00} STSLICECONST" /** - Slice size primitives. + Slice size primitives. */ /// Returns the number of references in a slice. @pure -fun getRemainingRefsCount(self: slice): int - asm "SREFS"; +fun slice.remainingRefsCount(self): int + asm "SREFS" /// Returns the number of data bits in a slice. @pure -fun getRemainingBitsCount(self: slice): int - asm "SBITS"; +fun slice.remainingBitsCount(self): int + asm "SBITS" /// Returns both the number of data bits and the number of references in a slice. @pure -fun getRemainingBitsAndRefsCount(self: slice): (int, int) - asm "SBITREFS"; +fun slice.remainingBitsAndRefsCount(self): (int, int) + asm "SBITREFS" /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure -fun isEndOfSlice(self: slice): bool - asm "SEMPTY"; +fun slice.isEmpty(self): bool + asm "SEMPTY" /// Checks whether a slice has no bits of data. @pure -fun isEndOfSliceBits(self: slice): bool - asm "SDEMPTY"; +fun slice.isEndOfBits(self): bool + asm "SDEMPTY" /// Checks whether a slice has no references. @pure -fun isEndOfSliceRefs(self: slice): bool - asm "SREMPTY"; +fun slice.isEndOfRefs(self): bool + asm "SREMPTY" /// Checks whether data parts of two slices coinside. @pure -fun isSliceBitsEqual(self: slice, b: slice): bool - asm "SDEQ"; +fun slice.bitsEqual(self, b: slice): bool + asm "SDEQ" /// Returns the number of cell references already stored in a builder. @pure -fun getBuilderRefsCount(self: builder): int - asm "BREFS"; +fun builder.refsCount(self): int + asm "BREFS" /// Returns the number of data bits already stored in a builder. @pure -fun getBuilderBitsCount(self: builder): int - asm "BBITS"; +fun builder.bitsCount(self): int + asm "BBITS" /** - Address manipulation primitives. - The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: - ```TL-B - addr_none$00 = MsgAddressExt; - addr_extern$01 len:(## 8) external_address:(bits len) - = MsgAddressExt; - anycast_info$_ depth:(#<= 30) { depth >= 1 } - rewrite_pfx:(bits depth) = Anycast; - addr_std$10 anycast:(Maybe Anycast) - workchain_id:int8 address:bits256 = MsgAddressInt; - addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) - workchain_id:int32 address:(bits addr_len) = MsgAddressInt; - _ _:MsgAddressInt = MsgAddress; - _ _:MsgAddressExt = MsgAddress; - - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ``` - A deserialized `MsgAddress` is represented by a tuple `t` as follows: - - - `addr_none` is represented by `t = (0)`, - i.e., a tuple containing exactly one integer equal to zero. - - `addr_extern` is represented by `t = (1, s)`, - where slice `s` contains the field `external_address`. In other words, ` - t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. - - `addr_std` is represented by `t = (2, u, x, s)`, - where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). - Next, integer `x` is the `workchain_id`, and slice `s` contains the address. - - `addr_var` is represented by `t = (3, u, x, s)`, - where `u`, `x`, and `s` have the same meaning as for `addr_std`. + Address manipulation primitives. */ -/// Loads from slice [s] the only prefix that is a valid `MsgAddress`, -/// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +/// Compile-time function that parses a valid contract address. +/// Example: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// Example: address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// Returns `address`, which can be stored in a builder, compared with `==`, etc. +@pure +fun address(stdAddress: slice): address + builtin + +/// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). +@pure +fun createAddressNone(): address + asm "b{00} PUSHSLICE" + +/// Returns if it's an empty address. +/// Don't confuse it with null! Empty address is a slice with two `0` bits. +/// In TL/B, it's addr_none$00. +@pure +fun address.isNone(self): bool + asm "b{00} SDBEGINSQ" "NIP" + +/// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). +/// All contract addresses are internal, so it's the most practical use case. +/// In TL/B it's addr_std$10. +/// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. +@pure +fun address.isInternal(self): bool + asm "b{10} SDBEGINSQ" "NIP" + +/// Returns if it's an external address, used to communication with the outside world. +/// In TL/B it's addr_extern$01. @pure -fun loadAddress(mutate self: slice): slice - asm( -> 1 0) "LDMSGADDR"; +fun address.isExternal(self): bool + asm "b{01} SDBEGINSQ" "NIP" -/// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. -/// If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +/// Extracts workchain and hash from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. @pure -fun parseAddress(s: slice): tuple - asm "PARSEMSGADDR"; +fun address.getWorkchainAndHash(self): (int8, uint256) + asm "REWRITESTDADDR" -/// Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), -/// applies rewriting from the anycast (if present) to the same-length prefix of the address, -/// and returns both the workchain and the 256-bit address as integers. -/// If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, -/// throws a cell deserialization exception. +/// Extracts workchain from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. @pure -fun parseStandardAddress(s: slice): (int, int) - asm "REWRITESTDADDR"; +fun address.getWorkchain(self): int8 + asm "REWRITESTDADDR" "DROP" -/// Creates a slice representing TL addr_none$00 (two `0` bits). +/// Checks whether two addresses are equal. Equivalent to `a == b`. +/// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. +/// Use just `a == b` and `a != b` to compare addresses, don't use bitsEqual. @pure -fun createAddressNone(): slice - asm "b{00} PUSHSLICE"; +@deprecated("use `senderAddress == ownerAddress`, not `senderAddress.bitsEqual(ownerAddress)`") +fun address.bitsEqual(self, b: address): bool + asm "SDEQ" -/// Returns if a slice pointer contains an empty address. -/// In other words, a slice starts with two `0` bits (TL addr_none$00). +/// Loads from slice [s] a valid `MsgAddress` (none/internal/external). @pure -fun addressIsNone(s: slice): bool - asm "2 PLDU" "0 EQINT"; +fun slice.loadAddress(mutate self): address + asm( -> 1 0) "LDMSGADDR" /** - Reserving Toncoins on balance and its flags. + Reserving Toncoins on balance and its flags. */ /// mode = 0: Reserve exact amount of nanotoncoins -const RESERVE_MODE_EXACT_AMOUNT = 0; +const RESERVE_MODE_EXACT_AMOUNT = 0 /// +1: Actually reserves all but amount, meaning `currentContractBalance - amount` -const RESERVE_MODE_ALL_BUT_AMOUNT = 1; +const RESERVE_MODE_ALL_BUT_AMOUNT = 1 /// +2: Actually set `min(amount, currentContractBalance)` (without this mode, if amount is greater, the action will fail) -const RESERVE_MODE_AT_MOST = 2; +const RESERVE_MODE_AT_MOST = 2 /// +4: [amount] is increased by the _original_ balance of the current account (before the compute phase). -const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4; +const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4 /// +8: Actually sets `amount = -amount` before performing any further actions. -const RESERVE_MODE_NEGATE_AMOUNT = 8; +const RESERVE_MODE_NEGATE_AMOUNT = 8 /// +16: If this action fails, the transaction will be bounced. -const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; +const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// Creates an output action which would reserve Toncoins on balance. /// For [reserveMode] consider constants above. -fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void - asm "RAWRESERVE"; +fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void + asm "RAWRESERVE" /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. -fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: cell?, reserveMode: int): void - asm "RAWRESERVEX"; +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, reserveMode: int): void + asm "RAWRESERVEX" /** - Messages sending and parsing primitives. - Working with messages is low-level right now, but still, every contract should do that. - - `Message` structure, its header and so on are specified in TL-B scheme, particularly: - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ... = CommonMsgInfo; + Creating and sending messages. + Basic scenario: + ``` + val outMsg = createMessage({ ... options }); // you get OutMessage + outMsg.send(mode); + ``` */ -/// 0b011000 tag - 0, ihr_disabled - 1, bounce - 1, bounced - 0, src = adr_none$00 -const BOUNCEABLE = 0x18; -/// 0b010000 tag - 0, ihr_disabled - 1, bounce - 0, bounced - 0, src = adr_none$00 -const NON_BOUNCEABLE = 0x10; +/// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. +const SEND_MODE_REGULAR = 0 +/// +1 means that the sender wants to pay transfer fees separately. +const SEND_MODE_PAY_FEES_SEPARATELY = 1 +/// +2 means that any errors arising while processing this message during the action phase should be ignored. +const SEND_MODE_IGNORE_ERRORS = 2 +/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16 +/// mode = 32 means that the current account must be destroyed if its resulting balance is zero. +const SEND_MODE_DESTROY = 32 +/// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. +const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64 +/// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). +const SEND_MODE_CARRY_ALL_BALANCE = 128 +/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 +/// Other modes affect the fee calculation as follows: +/// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). +/// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). + +type ExtraCurrenciesDict = dict + +/// ContractState is "code + data" of a contract. +/// Used in outgoing messages (StateInit) to initialize a destination contract. +struct ContractState { + code: cell + data: cell +} + +/// AddressShardingOptions provides settings to calculate an address in another shard. +/// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. +struct AddressShardingOptions { + fixedPrefixLength: uint5 // shard depth, formerly splitDepth + closeTo: address +} + +/// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. +/// In order to do this, it contains StateInit — and calculates an address by a hash of StateInit. +/// Example: +/// ``` +/// createMessage({ +/// dest: { +/// workchain: 0, +/// stateInit: { +/// code: jettonWalletCode, +/// data: jettonWalletEmptyStorage.toCell() +/// } +/// }, +/// ... +/// ``` +/// You just provide code+data, and the compiler automatically calculates the destination address, +/// because in TON, the address of a contract, by definition, is a hash of its initial state. +/// You can also use it without sending a message. See [buildAddress] and [addressMatches]. +struct AutoDeployAddress { + workchain: int8 = BASECHAIN + stateInit: ContractState | cell + toShard: AddressShardingOptions? = null +} + +/// Constructs an address that a deployed contract will have. +/// For example, from a jetton minter, you want to calculate an address of a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val jwAddrBuilt = jwDeployed.buildAddress(); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to calculate a jetton/nft address. +/// Note: returns `builder`, not `address`! It's cheap. +/// If you really need `address`, use `address.fromValidBuilder(res)`. +@pure +fun AutoDeployAddress.buildAddress(self): builder + builtin + +/// Checks that an address matches a deployed contract. +/// For example, from a jetton minter, you're checking whether a message is from a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val fromJW = jwDeployed.addressMatches(senderAddress); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. +@pure +fun AutoDeployAddress.addressMatches(self, addr: address): bool + builtin + +/// Options for creating an outgoing message. +/// Consider [createMessage] for examples. +struct CreateMessageOptions { + /// whether a message will bounce back on error + bounce: bool + /// message value: attached tons (or tons + extra currencies) + value: coins | (coins, ExtraCurrenciesDict) + /// destination is either a provided address, or is auto-calculated by stateInit + dest: address // either just send a message to some address + | builder // ... or a manually constructed builder with a valid address + | (int8, uint256) // ... or to workchain + hash (also known as accountID) + | AutoDeployAddress // ... or "send to stateInit" aka deploy (address auto-calculated) + /// body is any serializable object (or just miss this field for empty body) + body: TBody +} + +/// Creates a message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val reply = createMessage({ +/// bounce: false, +/// value: ton("0.05"), +/// dest: senderAddress, +/// body: RequestedInfo { ... } // note: no `toCell`! just pass an object +/// }); +/// reply.send(SEND_MODE_REGULAR); +/// ``` +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +/// If you need an empty body, just miss the field `body`, fill other 3 fields. +@pure +fun createMessage(options: CreateMessageOptions): OutMessage + builtin + + +/// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". +/// It includes some "topic" (arbitrary number), that determines the format of the message body. +/// For example, you emit "deposit event" (reserving topic "deposit" = 123): +/// > `dest: ExtOutLogBucket { topic: 123 }, body: DepositData { ... }` +/// and external indexers can index your emitted logs by destination address without parsing body. +/// Currently, external messages are used only for emitting logs (for viewing them in indexers). +/// In the future, there might be +/// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } +/// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total +struct (0x00) ExtOutLogBucket { + topic: uint248 | bits248 +} + +/// Options for creating an external outgoing message. +/// Consider [createExternalLogMessage] for examples. +struct CreateExternalLogMessageOptions { + /// destination is either an external address or a pattern to calculate it + dest: address // either some valid external/none address (not internal!) + | builder // ... or a manually constructed builder with a valid external address + | ExtOutLogBucket // ... or encode topic/eventID in destination + /// body is any serializable object (or just miss this field for empty body) + body: TBody +} + +/// Creates an external message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val emitMsg = createExternalLogMessage({ +/// dest: createAddressNone(), +/// body: DepositEvent { ... } // note: no `toCell`! just pass an object +/// }); +/// emitMsg.send(SEND_MODE_REGULAR); +/// ``` +/// Note: `createMessage` also returns `OutMessage`, it's okay: a composed message cell is universal. +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +@pure +fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage + builtin + +/// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, +/// it's packed into a separate ref. Wrapping body with this struct tells the compiler: +/// "inline message body in-place, I guarantee it will fit". +/// Example: +/// ``` +/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } // max 620 bits +/// +/// val contents: ProbablyLarge = { ... }; // you are sure: values are small +/// createMessage({ +/// // body: contents, // by default, it will be stored a ref (it's large) +/// body: UnsafeBodyNoRef { +/// forceInline: contents, // but wrapping forces the compiler to inline it +/// } +/// ``` +/// Another example: your body contains `builder` or `RemainingBitsAndRefs`: unpredictable size. +/// If you guarantee it's small and refs won't clash with code/data, avoid creating a cell. +struct UnsafeBodyNoRef { + forceInline: T +} + +/// OutMessage is a result of [createMessage]. +/// Essentially, it's a composed message cell, ready to be sent. +struct OutMessage { + messageCell: cell +} + +/// Sends a ready message cell. +/// For `sendMode`, see the constants above (SEND_MODE_*). +fun OutMessage.send(self, sendMode: int): void + asm "SENDRAWMSG" + +fun OutMessage.sendAndEstimateFee(self, sendMode: int): coins + asm "SENDMSG" + +@pure +fun OutMessage.estimateFeeWithoutSending(self, sendMode: int): coins + asm "10 PUSHPOW2" "OR" "SENDMSG" + +@pure +fun OutMessage.hash(self): uint256 + asm "HASHCU" + + +/// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. +/// But for everyday tasks, it's too complicated. It is not used in practice. +/// To represent code+data, consider [ContractState]. +/// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. +struct StateInit { + fixedPrefixLength: uint5? + special: (bool, bool)? + code: cell? + data: cell? + library: cell? +} + +/// Calculates a hash of StateInit if only code+data are set. +/// Example: +/// ``` +/// val addrHash = StateInit.calcHashCodeData(codeCell, dataCell); +/// createMessage({ +/// dest: (workchain, addrHash), +/// ... +/// ``` +@pure +fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm +""" // code data + DUP2 // code data code data + HASHCU + SWAP + HASHCU // code data dataHash codeHash + SWAP2 // dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // dataHash codeHash dataDepth codeDepth + + NEWC + x{020134} STSLICECONST // store refs_descriptor | bits_descriptor | data + 16 STU // store codeDepth + 16 STU // store dataDepth + 256 STU // store codeHash + 256 STU // store dataHash + + ONE HASHEXT_SHA256 +""" + +/// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. +@pure +fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: cell): uint256 asm +""" // pDepth code data + DUP2 // pDepth code data code data + HASHCU + SWAP + HASHCU // pDepth code data dataHash codeHash + SWAP2 // pDepth dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // pDepth dataHash codeHash dataDepth codeDepth + + s4 PUSH // pDepth dataHash codeHash dataDepth codeDepth pDepth + 10 LSHIFT# + 0x020381A0 PUSHINT + ADD // calc refs_descriptor | bits_descriptor | data + + NEWC + 32 STU + 16 STU + 16 STU + 256 STU + 256 STU + + ONE HASHEXT_SHA256 // pDepth hash + NIP +""" + + +/// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). +/// Example for fixedPrefixLength (shard depth) = 8: +/// | self (A) | aaaaaaaaaaa...aaa | +/// | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) +/// | result | 01010101aaa...aaa | address of A in same shard as B +/// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". +/// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). +/// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: +/// ``` +/// createMessage({ +/// dest: resBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... +/// ``` +/// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. +@pure +fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder + builtin + +/// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). +/// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. +/// Example: you've manually built a 267-bit address: '100' (std addr no anycast) + workchain + hash +/// and you want to return `address` (TVM slice) from a contract getter: +/// ``` +/// get calcWalletAddress(ownerAddress: address): address { +/// val b = calculate(...); // manually calculate into builder +/// return address.fromValidBuilder(b) // but really need address +/// } +/// ``` +/// Hint: if you want just to send a message, don't call this function, use a builder directly: +/// ``` +/// createMessage({ +/// dest: someBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... // so, don't call this expensive function in this case +/// ``` +@pure +fun address.fromValidBuilder(b: builder): address + asm "ENDC" "CTOS" + + +/// Sends a raw message — a correctly serialized TL object `Message X`. +/// In practice, you'll use a high-level wrapper [createMessage]. +fun sendRawMessage(msg: cell, mode: int): void + asm "SENDRAWMSG" -/// Load msgFlags from incoming message body (4 bits). -@pure -fun loadMessageFlags(mutate self: slice): int - asm( -> 1 0) "4 LDU"; -/// Having msgFlags (4 bits), check that a message is bounced. -/// Effectively, it's `msgFlags & 1` (the lowest bit present). -@pure -fun isMessageBounced(msgFlags: int): bool - asm "2 PUSHINT" "MODR"; + +/** + Receiving and handling messages. +*/ + +/// InMessage is an input for `onInternalMessage` — when your contract accepts a non-bounced message. +/// Internally, some data exist on the stack, some can be acquired with TVM instructions. +/// The compiler replaces `in.someField` and gets a value correctly. +/// Example: +/// ``` +/// fun onInternalMessage(in: InMessage) { +/// in.senderAddress // actually calls `INMSG_SRC` asm instruction +/// in.body // actually gets from a TVM stack +/// ``` +struct InMessage { + senderAddress: address // an internal address from which the message arrived + valueCoins: coins // ton amount attached to an incoming message + valueExtra: dict // extra currencies attached to an incoming message + originalForwardFee: coins // fee that was paid by the sender + createdLt: uint64 // logical time when a message was created + createdAt: uint32 // unixtime when a message was created + body: slice // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` +} + +/// InMessageBounced is an input for `onBouncedMessage`. +/// Very similar to a non-bounced input [InMessage]. +/// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. +/// Parse it with care! +/// Example: +/// ``` +/// fun onBouncedMessage(in: InMessageBounced) { +/// in.bouncedBody.skipBouncedPrefix(); +/// val originalOpcode = in.bouncedBody.loadUint(32); +/// ``` +struct InMessageBounced { + senderAddress: address // an internal address from which the message was bounced + valueCoins: coins // ton amount attached to a message + valueExtra: dict // extra currencies attached to a message + originalForwardFee: coins // comission that the sender has payed to send this message + createdLt: uint64 // logical time when a message was created (and bounced) + createdAt: uint32 // unixtime when a message was created (and bounced) + bouncedBody: slice // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally +} /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure -fun skipBouncedPrefix(mutate self: slice): self - asm "32 PUSHINT" "SDSKIPFIRST"; +fun slice.skipBouncedPrefix(mutate self): self + asm "32 LDU" "NIP" -/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +/// Load msgFlags from incoming message body (4 bits). @pure -fun loadMessageOp(mutate self: slice): int - asm( -> 1 0) "32 LDU"; +@deprecated("use `onBouncedMessage` handler instead of parsing msgCell flags manually") +fun slice.loadMessageFlags(mutate self): int + asm( -> 1 0) "4 LDU" +/// Loads a message opcode (32-bit integer) when parsing message body manually. @pure -fun skipMessageOp(mutate self: slice): self - asm "32 PUSHINT" "SDSKIPFIRST"; +fun slice.loadMessageOp(mutate self): int + asm( -> 1 0) "32 LDU" +/// Stores a message opcode (32-bit integer) when composing an output message manually. @pure -fun storeMessageOp(mutate self: builder, op: int): self - asm(op self) "32 STU"; +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageOp(mutate self, op: int): self + asm(op self) "32 STU" -/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +/// Loads a message queryId (64-bit integer, immediately following the opcode) when parsing message body manually. @pure -fun loadMessageQueryId(mutate self: slice): int - asm( -> 1 0) "64 LDU"; +@deprecated("use structures and lazy match instead of parsing messages manually") +fun slice.loadMessageQueryId(mutate self): int + asm( -> 1 0) "64 LDU" @pure -fun skipMessageQueryId(mutate self: slice): self - asm "64 PUSHINT" "SDSKIPFIRST"; +@deprecated("use structures and lazy loading instead of parsing messages manually") +fun slice.skipMessageQueryId(mutate self): self + asm "64 LDU" "NIP" @pure -fun storeMessageQueryId(mutate self: builder, queryId: int): self - asm(queryId self) "64 STU"; +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageQueryId(mutate self, queryId: int): self + asm(queryId self) "64 STU" -/// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG - -/// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. -const SEND_MODE_REGULAR = 0; -/// +1 means that the sender wants to pay transfer fees separately. -const SEND_MODE_PAY_FEES_SEPARATELY = 1; -/// +2 means that any errors arising while processing this message during the action phase should be ignored. -const SEND_MODE_IGNORE_ERRORS = 2; -/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16; -/// mode = 32 means that the current account must be destroyed if its resulting balance is zero. -const SEND_MODE_DESTROY = 32; -/// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. -const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64; -/// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). -const SEND_MODE_CARRY_ALL_BALANCE = 128; -/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; -/// Other modes affect the fee calculation as follows: -/// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). -/// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). - -/// Sends a raw message — a correctly serialized TL object `Message X`. -/// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). -/// This function is still available, but deprecated: consider using [sendMessage]. -@deprecated -fun sendRawMessage(msg: cell, mode: int): void - asm "SENDRAWMSG"; -/// Creates an output action and returns a fee for creating a message. -/// Mode has the same effect as in the case of SENDRAWMSG. -/// For mode including SEND_MODE_ESTIMATE_FEE_ONLY it just returns estimated fee without sending a message. -fun sendMessage(msg: cell, mode: int): int - asm "SENDMSG"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 9873ca94d..916cefc6c 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 1.0 /** Gas and payment related primitives. @@ -7,7 +7,7 @@ tolk 0.9 /// Returns amount of gas (in gas units) consumed in current Computation Phase. fun getGasConsumedAtTheMoment(): int - asm "GASCONSUMED"; + asm "GASCONSUMED" /// This function is required to be called when you process an external message (from an outer world) /// and "accept" it to blockchain. @@ -15,14 +15,14 @@ fun getGasConsumedAtTheMoment(): int /// As an effect, the current smart contract agrees to buy some gas to finish the current transaction. /// For more details, check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). fun acceptExternalMessage(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. /// Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, /// decreasing the value of `gr` by `gc` in the process. fun setGasLimitToMaximum(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. @@ -30,40 +30,41 @@ fun setGasLimitToMaximum(): void /// If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, /// an (unhandled) out of gas exception is thrown before setting new gas limits. fun setGasLimit(limit: int): void - asm "SETGASLIMIT"; + asm "SETGASLIMIT" /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. -fun calculateGasFee(workchain: int, gasUsed: int): int - asm(gasUsed workchain) "GETGASFEE"; +fun calculateGasFee(workchain: int8, gasUsed: int): coins + asm(gasUsed workchain) "GETGASFEE" /// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateGasFeeWithoutFlatPrice(workchain: int, gasUsed: int): int - asm(gasUsed workchain) "GETGASFEESIMPLE"; +fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins + asm(gasUsed workchain) "GETGASFEESIMPLE" /// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. /// [bits] and [cells] represent contract state (code + data). -fun calculateStorageFee(workchain: int, seconds: int, bits: int, cells: int): int - asm(cells bits seconds workchain) "GETSTORAGEFEE"; +fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins + asm(cells bits seconds workchain) "GETSTORAGEFEE" -/// Calculates amount of nanotoncoins you should pay to send a message of specified size. -fun calculateMessageFee(workchain: int, bits: int, cells: int): int - asm(cells bits workchain) "GETFORWARDFEE"; +/// Calculates amount of nanotoncoins you should pay to send a message of a specified size. +fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins + asm(cells bits workchain) "GETFORWARDFEE" -/// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateMessageFeeWithoutLumpPrice(workchain: int, bits: int, cells: int): int - asm(cells bits workchain) "GETFORWARDFEESIMPLE"; +/// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins + asm(cells bits workchain) "GETFORWARDFEESIMPLE" /// Calculates fee that was paid by the sender of an incoming internal message. -fun calculateOriginalMessageFee(workchain: int, incomingFwdFee: int): int - asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; +@deprecated("use modern `onInternalMessage` and access `in.originalForwardFee` directly") +fun calculateOriginalForwardFee(workchain: int8, incomingFwdFee: coins): coins + asm(incomingFwdFee workchain) "GETORIGINALFWDFEE" /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) /// If it has no debt, `0` is returned. -fun getMyStorageDuePayment(): int - asm "DUEPAYMENT"; +fun contract.getStorageDuePayment(): coins + asm "DUEPAYMENT" /// Returns the amount of nanotoncoins charged for storage. /// (during storage phase preceeding to current computation phase) @pure -fun getMyStoragePaidPayment(): int - asm "STORAGEFEES"; +fun contract.getStoragePaidPayment(): coins + asm "STORAGEFEES" diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index e63438b51..212586b1d 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,39 +1,33 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 1.0 /** - Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. + Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. Elements of a list can be of different types. Empty list is conventionally represented as TVM `null` value. */ @pure fun createEmptyList(): tuple - asm "PUSHNULL"; + asm "PUSHNULL" /// Adds an element to the beginning of lisp-style list. /// Note, that it does not mutate the list: instead, it returns a new one (it's a lisp pattern). @pure fun listPrepend(head: X, tail: tuple?): tuple - asm "CONS"; + asm "CONS" /// Extracts the head and the tail of lisp-style list. @pure fun listSplit(list: tuple): (X, tuple?) - asm "UNCONS"; - -/// Extracts the tail and the head of lisp-style list. -/// After extracting the last element, tuple is assigned to null. -@pure -fun listNext(mutate self: tuple?): X - asm( -> 1 0) "UNCONS"; + asm "UNCONS" /// Returns the head of lisp-style list. @pure fun listGetHead(list: tuple): X - asm "CAR"; + asm "CAR" /// Returns the tail of lisp-style list. @pure fun listGetTail(list: tuple): tuple? - asm "CDR"; + asm "CDR" diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index ee2056874..41e091f0a 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 1.0 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). @@ -13,286 +13,290 @@ tolk 0.9 Every dictionary object (`self` parameter) can be null. TVM NULL is essentially "empty dictionary". */ +/// In @stdlib/common.tolk, there is a type alias: +/// `type dict = cell?` +/// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. + /// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL @pure -fun createEmptyDict(): cell? - asm "NEWDICT"; +fun createEmptyDict(): dict + asm "NEWDICT" /// Checks whether a dictionary is empty. @pure -fun dictIsEmpty(self: cell?): bool - asm "DICTEMPTY"; +fun dict.dictIsEmpty(self): bool + asm "DICTEMPTY" @pure -fun iDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; +fun dict.iDictGet(self, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT" @pure -fun uDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; +fun dict.uDictGet(self, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT" @pure -fun sDictGet(self: cell?, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; +fun dict.sDictGet(self, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT" @pure -fun iDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTISET"; +fun dict.iDictSet(mutate self, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTISET" @pure -fun uDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTUSET"; +fun dict.uDictSet(mutate self, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTUSET" @pure -fun sDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): void - asm(value key self keyLen) "DICTSET"; +fun dict.sDictSet(mutate self, keyLen: int, key: slice, value: slice): void + asm(value key self keyLen) "DICTSET" @pure -fun iDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTISETREF"; +fun dict.iDictSetRef(mutate self, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTISETREF" @pure -fun uDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTUSETREF"; +fun dict.uDictSetRef(mutate self, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTUSETREF" @pure -fun sDictSetRef(mutate self: cell?, keyLen: int, key: slice, value: cell): void - asm(value key self keyLen) "DICTSETREF"; +fun dict.sDictSetRef(mutate self, keyLen: int, key: slice, value: cell): void + asm(value key self keyLen) "DICTSETREF" @pure -fun iDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIADD"; +fun dict.iDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIADD" @pure -fun uDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUADD"; +fun dict.uDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUADD" @pure -fun iDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIREPLACE"; +fun dict.iDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIREPLACE" @pure -fun uDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUREPLACE"; +fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUREPLACE" @pure -fun iDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) - asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; +fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) + asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT" @pure -fun uDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) - asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; +fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) + asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT" @pure -fun sDictGetRef(self: cell?, keyLen: int, key: slice): (cell?, bool) - asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; +fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) + asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT" @pure -fun iDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? - asm(key self keyLen) "DICTIGETOPTREF"; +fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict + asm(key self keyLen) "DICTIGETOPTREF" @pure -fun uDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? - asm(key self keyLen) "DICTUGETOPTREF"; +fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict + asm(key self keyLen) "DICTUGETOPTREF" @pure -fun sDictGetRefOrNull(self: cell?, keyLen: int, key: slice): cell? - asm(key self keyLen) "DICTGETOPTREF"; +fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict + asm(key self keyLen) "DICTGETOPTREF" @pure -fun iDictDelete(mutate self: cell?, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; +fun dict.iDictDelete(mutate self, keyLen: int, key: int): bool + asm(key self keyLen) "DICTIDEL" @pure -fun uDictDelete(mutate self: cell?, keyLen: int, key: int): bool - asm(key self keyLen) "DICTUDEL"; +fun dict.uDictDelete(mutate self, keyLen: int, key: int): bool + asm(key self keyLen) "DICTUDEL" @pure -fun sDictDelete(mutate self: cell?, keyLen: int, key: slice): bool - asm(key self keyLen) "DICTDEL"; +fun dict.sDictDelete(mutate self, keyLen: int, key: slice): bool + asm(key self keyLen) "DICTDEL" @pure -fun iDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; +fun dict.iDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT" @pure -fun uDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; +fun dict.uDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT" @pure -fun sDictSetAndGet(mutate self: cell?, keyLen: int, key: slice, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; +fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT" @pure -fun iDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? - asm(value key self keyLen) "DICTISETGETOPTREF"; +fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict + asm(value key self keyLen) "DICTISETGETOPTREF" @pure -fun uDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? - asm(value key self keyLen) "DICTUSETGETOPTREF"; +fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict + asm(value key self keyLen) "DICTUSETGETOPTREF" @pure -fun iDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; +fun dict.iDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT" @pure -fun uDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; +fun dict.uDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT" @pure -fun sDictDeleteAndGet(mutate self: cell?, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; +fun dict.sDictDeleteAndGet(mutate self, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT" @pure -fun iDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTISETB"; +fun dict.iDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTISETB" @pure -fun uDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTUSETB"; +fun dict.uDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTUSETB" @pure -fun sDictSetBuilder(mutate self: cell?, keyLen: int, key: slice, value: builder): void - asm(value key self keyLen) "DICTSETB"; +fun dict.sDictSetBuilder(mutate self, keyLen: int, key: slice, value: builder): void + asm(value key self keyLen) "DICTSETB" @pure -fun iDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIADDB"; +fun dict.iDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIADDB" @pure -fun uDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUADDB"; +fun dict.uDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUADDB" @pure -fun iDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIREPLACEB"; +fun dict.iDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIREPLACEB" @pure -fun uDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUREPLACEB"; +fun dict.uDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUREPLACEB" @pure -fun iDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; +fun dict.iDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2" @pure -fun uDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; +fun dict.uDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2" @pure -fun sDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; +fun dict.sDictDeleteFirstAndGet(mutate self, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2" @pure -fun iDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; +fun dict.iDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2" @pure -fun uDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; +fun dict.uDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2" @pure -fun sDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; +fun dict.sDictDeleteLastAndGet(mutate self, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2" @pure -fun iDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; +fun dict.iDictGetFirst(self, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2" @pure -fun uDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; +fun dict.uDictGetFirst(self, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2" @pure -fun sDictGetFirst(self: cell?, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; +fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2" @pure -fun iDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) - asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; +fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) + asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2" @pure -fun uDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) - asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; +fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) + asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2" @pure -fun sDictGetFirstAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) - asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; +fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) + asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2" @pure -fun iDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; +fun dict.iDictGetLast(self, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2" @pure -fun uDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; +fun dict.uDictGetLast(self, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2" @pure -fun sDictGetLast(self: cell?, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; +fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2" @pure -fun iDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) - asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; +fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) + asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2" @pure -fun uDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) - asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; +fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) + asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2" @pure -fun sDictGetLastAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) - asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; +fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) + asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2" @pure -fun iDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; +fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2" @pure -fun uDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; +fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2" @pure -fun iDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; +fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2" @pure -fun uDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; +fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2" @pure -fun iDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; +fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2" @pure -fun uDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; +fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2" @pure -fun iDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; +fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2" @pure -fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; +fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2" /** @@ -300,13 +304,13 @@ fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bo */ @pure -fun prefixDictGet(self: cell?, keyLen: int, key: slice): (slice, slice?, slice?, bool) - asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; +fun dict.prefixDictGet(self, keyLen: int, key: slice): (slice, slice?, slice?, bool) + asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2" @pure -fun prefixDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): bool - asm(value key self keyLen) "PFXDICTSET"; +fun dict.prefixDictSet(mutate self, keyLen: int, key: slice, value: slice): bool + asm(value key self keyLen) "PFXDICTSET" @pure -fun prefixDictDelete(mutate self: cell?, keyLen: int, key: slice): bool - asm(key self keyLen) "PFXDICTDEL"; +fun dict.prefixDictDelete(mutate self, keyLen: int, key: slice): bool + asm(key self keyLen) "PFXDICTDEL" diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 136eaa4a1..d578199c7 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,25 +1,25 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 1.0 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. @pure fun getTvmRegisterC3(): continuation - asm "c3 PUSH"; + asm "c3 PUSH" /// Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. /// Note that after execution of this primitive the current code /// (and the stack of recursive function calls) won't change, /// but any other function call will use a function from the new code. fun setTvmRegisterC3(c: continuation): void - asm "c3 POP"; + asm "c3 POP" /// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. @pure fun transformSliceToContinuation(s: slice): continuation - asm "BLESS"; + asm "BLESS" /// Moves a variable or a value [x] to the top of the stack. @pure -fun stackMoveToTop(mutate self: X): void - asm "NOP"; +fun T.stackMoveToTop(mutate self): void + asm "NOP" diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 4d860fbab..c067b88ff 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -19,6 +19,7 @@ #include "SmartContract.h" #include "GenericAccount.h" +#include "transaction.h" #include "block/block.h" #include "block/block-auto.h" @@ -60,9 +61,9 @@ td::Ref build_internal_message(td::RefInt256 amount, td::Refbit_size(false) + 7) >> 3); b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams - b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init // body:(Either X ^X) - if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + if (b.remaining_bits() >= 1 + body->size() && b.remaining_refs() >= body->size_refs()) { b.store_zeroes(1); b.append_cellslice(body); } else { @@ -161,7 +162,9 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod vm::load_cell_slice_ref(address), // myself:MsgAddressInt vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) }; - if (args.config && args.config.value()->get_global_version() >= 4) { + + int global_version = args.config ? args.config.value()->get_global_version() : SUPPORTED_VERSION; + if (global_version >= 4) { tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] tuple.push_back(td::zero_refint()); // storage_fees:Integer @@ -172,21 +175,61 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod // prev_key_block:BlockId ] : PrevBlocksInfo tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info } - if (args.config && args.config.value()->get_global_version() >= 6) { - tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple + if (global_version >= 6) { + tuple.push_back(args.config ? args.config.value()->get_unpacked_config_tuple(now) + : vm::StackEntry{}); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; - if (code.not_null()) { + if (code.not_null() && args.config) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (global_version >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); } +std::shared_ptr try_fetch_config_from_c7(td::Ref c7) { + if (c7.is_null() || c7->size() < 1) { + return nullptr; + } + auto c7_tuple = c7->at(0).as_tuple(); + if (c7_tuple.is_null() || c7_tuple->size() < 10) { + return nullptr; + } + auto config_cell = c7_tuple->at(9).as_cell(); + if (config_cell.is_null()) { + return nullptr; + } + auto config_dict = std::make_unique(config_cell, 32); + auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); + ton::StdSmcAddress config_addr; + if (config_addr_cell.is_null()) { + config_addr = ton::StdSmcAddress::zero(); + } else { + auto config_addr_cs = vm::load_cell_slice(std::move(config_addr_cell)); + if (config_addr_cs.size() != 0x100) { + LOG(WARNING) << "Config parameter 0 with config address has wrong size"; + config_addr = ton::StdSmcAddress::zero(); + } else { + config_addr_cs.fetch_bits_to(config_addr); + } + } + auto global_config = block::Config(config_cell, std::move(config_addr), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto unpack_res = global_config.unpack(); + if (unpack_res.is_error()) { + LOG(ERROR) << "Failed to unpack config: " << unpack_res.error(); + return nullptr; + } + return std::make_shared(std::move(global_config)); +} + SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, int vm_log_verbosity, bool debug_enabled, @@ -223,7 +266,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - int global_version = config ? config->get_global_version() : 0; + int global_version = config ? config->get_global_version() : SUPPORTED_VERSION; vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig); @@ -310,6 +353,9 @@ td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } @@ -331,6 +377,9 @@ SmartContract::Answer SmartContract::run_method(Args args) { } SmartContract::Answer SmartContract::run_get_method(Args args) const { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 131bfe92e..940b50796 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "crypto/Ed25519.h" -#include "ellcurve/Ed25519.h" #include "td/utils/logging.h" #include "td/utils/misc.h" diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 9a92e0b1e..55a6ded75 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -171,3 +171,11 @@ TEST(Fift, test_levels) { TEST(Fift, test_secp256k1) { run_fift("secp256k1.fif"); } + +TEST(Fift, test_get_extra_balance) { + run_fift("get_extra_balance.fif"); +} + +TEST(Fift, test_p256) { + run_fift("p256.fif"); +} diff --git a/crypto/test/fift/get_extra_balance.fif b/crypto/test/fift/get_extra_balance.fif new file mode 100644 index 000000000..2b475a7ed --- /dev/null +++ b/crypto/test/fift/get_extra_balance.fif @@ -0,0 +1,80 @@ +"Asm.fif" include +"FiftExt.fif" include +"TonUtil.fif" include + +null constant extras +{ null } 7 times +10 extras 2 tuple +8 tuple 1 tuple constant c7 +<{ + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 7 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 8 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 9 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 10 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew 1 +{ + =: idx =: d + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times diff --git a/crypto/test/fift/p256.fif b/crypto/test/fift/p256.fif new file mode 100644 index 000000000..ecd43d19b --- /dev/null +++ b/crypto/test/fift/p256.fif @@ -0,0 +1,25 @@ +"Asm.fif" include +"FiftExt.fif" include + +// Test vectors from https://datatracker.ietf.org/doc/html/rfc6979#appendix-A.2.5 + +x{73616d706c65} // "sample" +x{EFD48B2AACB6A8FD1140DD9CD45E81D69D2C877B56AAF991C34D0EA84EAF3716F7CB1C942D657C41D436C7A1B6E29F65F3E900DBB9AFF4064DC4AB2F843ACDA8} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{F1ABB023518351CD71D881567B1EA663ED3EFCF6C5132B354F28D3B0B7D38367019F4113742A2B14BD25926B49C649155F267E60D3814B4C0CC84250E46F0083} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{83910E8B48BB0C74244EBDF7F07A1C5413D61472BD941EF3920E623FBCCEBEB68DDBEC54CF8CD5874883841D712142A56A8D0F218F5003CB0296B6B509619F2C} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // 0 0 +drop drop diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index dc7fcf370..c70738c7b 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -17,14 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/boc.h" -#include "vm/cellslice.h" #include "vm/cells.h" #include "common/AtomicRef.h" #include "vm/cells/CellString.h" #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" #include "vm/db/CellStorage.h" -#include "vm/db/CellHashTable.h" #include "vm/db/TonDb.h" #include "vm/db/StaticBagOfCellsDb.h" @@ -40,7 +38,6 @@ #include "td/utils/port/path.h" #include "td/utils/format.h" #include "td/utils/misc.h" -#include "td/utils/optional.h" #include "td/utils/tests.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" @@ -50,93 +47,94 @@ #include "td/db/MemoryKeyValue.h" #include "td/db/utils/CyclicBuffer.h" -#include "td/fec/fec.h" - #include #include #include +#include #include #include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/utils/VectorQueue.h" #include "vm/dict.h" -#include #include #include #include -#include +#include -namespace vm { -class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { +#include +#include +#include + +#include "td/actor/actor.h" +#include "td/utils/overloaded.h" + +class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: - explicit ThreadExecutor(size_t threads_n) { - for (size_t i = 0; i < threads_n; ++i) { - threads_.emplace_back([this]() { - while (true) { - auto task = pop_task(); - if (!task) { - break; - } - CHECK(generation_.load() % 2 == 1); - task(); - } - }); - } + ActorExecutor(size_t tn) : tn_(tn) { + scheduler_.run_in_context([&] { worker_ = td::actor::create_actor("Worker"); }); + thread_ = td::thread([this]() { scheduler_.run(); }); } - - ~ThreadExecutor() override { - for (size_t i = 0; i < threads_.size(); ++i) { - push_task({}); + ~ActorExecutor() { + scheduler_.run_in_context_external([&] { send_closure(worker_, &Worker::close); }); + thread_.join(); + } + std::string describe() const override { + return PSTRING() << "ActorExecutor(tn=" << tn_ << ")"; + } + class Worker : public td::actor::Actor { + public: + void close() { + td::actor::core::SchedulerContext::get()->stop(); + stop(); } - for (auto &t : threads_) { - t.join(); + void execute_sync(std::function f) { + f(); } - } + }; void execute_async(std::function f) override { - push_task(std::move(f)); + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { + f_(); + stop(); + } + + private: + std::function f_; + }; + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::create_actor("executeasync", std::move(f)).release(); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::create_actor("executeasync", std::move(f)).release(); }); + } } void execute_sync(std::function f) override { - auto x = generation_.load(); - std::scoped_lock lock(sync_mutex_); - CHECK(x == generation_); - CHECK(generation_.load() % 2 == 1); - f(); - CHECK(generation_.load() % 2 == 1); - } - void inc_generation() { - generation_.fetch_add(1); + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); }); + } } private: - std::atomic generation_{0}; - std::queue, size_t>> queue_; - std::mutex queue_mutex_; - std::condition_variable cv_; - std::mutex sync_mutex_; - std::vector threads_; - - std::function pop_task() { - std::unique_lock lock(queue_mutex_); - cv_.wait(lock, [&] { return !queue_.empty(); }); - CHECK(!queue_.empty()); - auto task = std::move(queue_.front()); - queue_.pop(); - CHECK(task.second == generation_); - return task.first; - } - - void push_task(std::function task) { - { - std::scoped_lock lock(queue_mutex_); - queue_.emplace(std::move(task), generation_.load()); - } - cv_.notify_one(); - } + size_t tn_; + td::actor::Scheduler scheduler_{{tn_}, false, td::actor::Scheduler::Paused}; + td::actor::ActorOwn worker_; + td::thread thread_; }; +namespace vm { std::vector do_get_serialization_modes() { std::vector res; for (int i = 0; i < 32; i++) { @@ -324,6 +322,34 @@ TEST(Cell, sha_benchmark_threaded) { bench_threaded([n]() { return BenchSha256(n); }); } } +class BenchTasks : public td::Benchmark { + public: + explicit BenchTasks(size_t tn) : tn_(tn) { + } + + std::string get_description() const override { + return PSTRING() << "bench_tasks(threads_n=" << tn_ << ")"; + } + + void run(int n) override { + ActorExecutor executor(tn_); + for (int i = 0; i < n; i++) { + std::latch latch(tn_); + for (size_t j = 0; j < tn_; j++) { + executor.execute_async([&]() { latch.count_down(); }); + } + latch.wait(); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, Tasks) { + for (size_t tn : {1, 4, 16}) { + bench(BenchTasks(tn)); + } +} std::string serialize_boc(Ref cell, int mode = 31) { CHECK(cell.not_null()); @@ -437,6 +463,8 @@ class CellExplorer { cs_ = {}; break; } + default: + UNREACHABLE(); } } @@ -474,6 +502,8 @@ class CellExplorer { case op.ReadCellSlice: log_ << "read slice " << op.children_mask << "\n"; break; + default: + UNREACHABLE(); } } void log_cell(const Ref &cell) { @@ -627,7 +657,9 @@ TEST(Cell, MerkleProof) { auto exploration2 = CellExplorer::explore(usage_cell, exploration.ops); ASSERT_EQ(exploration.log, exploration2.log); - auto is_prunned = [&](const Ref &cell) { return exploration.visited.count(cell->get_hash()) == 0; }; + auto is_prunned = [&](const Ref &cell_to_check) { + return exploration.visited.count(cell_to_check->get_hash()) == 0; + }; auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); @@ -706,7 +738,7 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - auto cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12, 1); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); @@ -927,7 +959,6 @@ TEST(TonDb, InMemoryDynamicBocSimple) { auto before = counter(); SCOPE_EXIT { LOG_CHECK(before == counter()) << before << " vs " << counter(); - ; }; td::Random::Xorshift128plus rnd{123}; auto kv = std::make_shared(); @@ -963,26 +994,199 @@ TEST(TonDb, InMemoryDynamicBocSimple) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; +struct CellMerger : td::Merger { + void merge_value_and_update(std::string &value, td::Slice update) override { + return CellStorer::merge_value_and_refcnt_diff(value, update); + } + void merge_update_and_update(std::string &left_update, td::Slice right_update) override { + LOG(ERROR) << "update_and_update"; + UNREACHABLE(); + return CellStorer::merge_refcnt_diffs(left_update, right_update); + } +}; +struct CompactionFilterEraseEmptyValues : public rocksdb::CompactionFilter { + bool Filter(int level, const rocksdb::Slice & /*key*/, const rocksdb::Slice &existing_value, std::string *new_value, + bool *value_changed) const override { + return existing_value.empty(); + } + bool FilterMergeOperand(int, const rocksdb::Slice & /*key*/, const rocksdb::Slice &operand) const override { + return operand.empty(); + } + + // Name of the compaction filter + const char *Name() const override { + return "CompactionFilterEraseEmptyValues"; + } +}; +auto to_td(rocksdb::Slice value) -> td::Slice { + return td::Slice(value.data(), value.size()); +} + +struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { + const char *Name() const override { + return "MergeOperatorAddCellRefcnt"; + } + bool FullMergeV2(const MergeOperationInput &merge_in, MergeOperationOutput *merge_out) const override { + CHECK(merge_in.existing_value); + auto &value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice & /*key*/, const rocksdb::Slice &left, const rocksdb::Slice &right, + std::string *new_value, rocksdb::Logger *logger) const override { + *new_value = left.ToString(); + CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +struct DB { + std::unique_ptr dboc; + std::shared_ptr kv; + void reset_loader() { + dboc->set_loader(std::make_unique(kv->snapshot())); + } +}; struct BocOptions { - std::shared_ptr async_executor; - std::optional o_in_memory; + using AsyncExecutor = DynamicBagOfCellsDb::AsyncExecutor; + + using CreateInMemoryOptions = DynamicBagOfCellsDb::CreateInMemoryOptions; + using CreateV1Options = DynamicBagOfCellsDb::CreateV1Options; + using CreateV2Options = DynamicBagOfCellsDb::CreateV2Options; + + std::shared_ptr async_executor; + struct KvOptions { + enum KvType { InMemory, RocksDb } kv_type{InMemory}; + bool experimental{false}; + bool no_transactions{false}; + size_t cache_size{0}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const KvOptions &kv_options) { + if (kv_options.kv_type == KvType::InMemory) { + return sb << "InMemory{}"; + } + return sb << "RockDb{cache_size=" << kv_options.cache_size << ", no_transactions=" << kv_options.no_transactions + << ", experimental=" << kv_options.experimental << "}"; + } + }; + KvOptions kv_options; + std::variant options; + std::pair compress_depth_range{0, 0}; td::uint64 seed{123}; + td::Random::Xorshift128plus rnd{123}; - auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { - if (o_in_memory) { - auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); + std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { + if (kv_options.kv_type == KvOptions::InMemory) { + if (old_key_value) { + return old_key_value; + } + return std::make_shared(std::make_shared()); + } else if (kv_options.kv_type == KvOptions::RocksDb) { + auto merge_operator = std::make_shared(); + static const CompactionFilterEraseEmptyValues compaction_filter; + CHECK(!old_key_value || old_key_value.use_count() == 1); + std::string db_path = "test_celldb"; + if (old_key_value) { + //LOG(ERROR) << "Reload rocksdb"; + old_key_value.reset(); + } else { + //LOG(ERROR) << "New rocksdb"; + td::RocksDb::destroy(db_path).ensure(); } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; - return res; + auto db_options = td::RocksDbOptions{ + .block_cache = {}, + .merge_operator = merge_operator, + .compaction_filter = &compaction_filter, + .experimental = kv_options.experimental, + .no_reads = no_reads, + .no_transactions = kv_options.no_transactions, + .use_direct_reads = true, + .no_block_cache = true, + }; + if (kv_options.cache_size != 0) { + db_options.no_block_cache = false; + db_options.block_cache = rocksdb::NewLRUCache(kv_options.cache_size); + } + return std::make_shared(td::RocksDb::open(db_path, std::move(db_options)).move_as_ok()); + } else { + UNREACHABLE(); + } + } + void check_kv_is_empty(KeyValue &kv) { + if (kv_options.kv_type == KvOptions::InMemory) { + ASSERT_EQ(0u, kv.count("").move_as_ok()); + return; + } + + size_t non_empty_values = 0; + kv.for_each([&](auto key, auto value) { + non_empty_values += !value.empty(); + return td::Status::OK(); + }); + if (non_empty_values != 0) { + kv.for_each([&](auto key, auto value) { + LOG(ERROR) << "Key: " << td::hex_encode(key) << " Value: " << td::hex_encode(value); + std::string x; + LOG(ERROR) << int(kv.get(key, x).move_as_ok()); + return td::Status::OK(); + }); + } + ASSERT_EQ(0u, non_empty_values); + } + + [[nodiscard]] auto create_db(DB db, std::optional o_root_n) { + auto old_boc = std::move(db.dboc); + auto old_kv = std::move(db.kv); + old_boc.reset(); + using ResT = DB; + auto res = std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); + if (compress_depth_range.second != 0) { + res.dboc->set_celldb_compress_depth(rnd.fast(compress_depth_range.first, compress_depth_range.second)); } - return DynamicBagOfCellsDb::create(); + return res; }; void prepare_commit(DynamicBagOfCellsDb &dboc) { + td::PerfWarningTimer warning_timer("test_db_prepare_commit"); if (async_executor) { - async_executor->inc_generation(); std::latch latch(1); td::Result res; async_executor->execute_sync([&] { @@ -993,70 +1197,191 @@ struct BocOptions { }); latch.wait(); async_executor->execute_sync([&] {}); - async_executor->inc_generation(); + res.ensure(); } else { dboc.prepare_commit(); } } + enum CacheAction { ResetCache, KeepCache }; + void write_commit(DynamicBagOfCellsDb &dboc, std::shared_ptr kv, CacheAction action) { + td::PerfWarningTimer warning_timer("test_db_write_commit"); + kv->begin_write_batch().ensure(); + CellStorer cell_storer(*kv); + { + td::PerfWarningTimer timer("test_db_commit"); + dboc.commit(cell_storer).ensure(); + } + { + td::PerfWarningTimer timer("test_db_commit_write_batch"); + kv->commit_write_batch().ensure(); + } + switch (action) { + case ResetCache: { + td::PerfWarningTimer timer("test_db_reset_cache"); + dboc.set_loader(std::make_unique(kv->snapshot())); + break; + } + case KeepCache: + break; + } + } + + void commit(DB &db, CacheAction action = ResetCache) { + prepare_commit(*db.dboc); + write_commit(*db.dboc, db.kv, action); + } + + std::string description() const { + td::StringBuilder sb; + + sb << "DBOC(type="; + std::visit(td::overloaded([&](const CreateV1Options &) { sb << "V1"; }, + [&](const CreateV2Options &options) { + sb << "V2(concurrency=" << options.extra_threads + 1; + if (options.executor) { + sb << ", executor=" << options.executor->describe(); + } else { + sb << ", executor=threads"; + } + sb << ")"; + }, + [&](const CreateInMemoryOptions &options) { + sb << "InMemory(use_arena=" << options.use_arena + << ", less_memory=" << options.use_less_memory_during_creation << ")"; + }), + options); + sb << kv_options; + if (async_executor) { + sb << ", executor=" << async_executor->describe(); + } + if (compress_depth_range.second != 0) { + sb << ", compress_depth=[" << compress_depth_range.first << ";" << compress_depth_range.second << "]"; + } + sb << ")"; + + return sb.as_cslice().str(); + } }; template -void with_all_boc_options(F &&f, size_t tests_n = 500) { +void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { LOG(INFO) << "Test dynamic boc"; auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + std::map>> benches; auto run = [&](BocOptions options) { - LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); - if (options.o_in_memory) { - LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena - << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; - } + auto description = options.description(); + LOG(INFO) << "Running " << description; + auto start = td::Timestamp::now(); + DynamicBagOfCellsDb::Stats stats; + auto o_in_memory = std::get_if(&options.options); for (td::uint32 i = 0; i < tests_n; i++) { auto before = counter(); + options.seed = i == 0 ? 123 : i; - f(options); + options.rnd = td::Random::Xorshift128plus{options.seed}; + auto stats_diff = f(options); + stats.apply_diff(stats_diff); + auto after = counter(); - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) - << before << " vs " << after; + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == after) << before << " vs " << after; + } + LOG(INFO) << "\ttook " << td::Timestamp::now().at() - start.at() << " seconds"; + LOG(INFO) << stats; + for (auto &[key, value] : stats.named_stats.stats_int) { + if (td::begins_with(key, "bench_")) { + benches[key].emplace_back(value, description); + } } }; - run({.async_executor = std::make_shared(4)}); - run({}); - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.o_in_memory = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + + // NB: use .experimental to play with different RocksDb parameters + // Note, that new benchmark are necessary to fully understand the effect of different RocksDb options + std::vector kv_options_list = { + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::InMemory}, + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = 0}, + BocOptions::KvOptions{ + .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, + }; + std::vector> compress_depth_ranges = {{0, 5}, {5, 5}, {0, 0}}; + std::vector has_executor_options = {false, true}; + for (auto compress_depth_range : compress_depth_ranges) { + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({ + .async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 3, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range, + }); + + // V1 + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV1Options{}, + .compress_depth_range = compress_depth_range}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range}); + + // InMemory + if (compress_depth_range.second == 0) { + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } + } + } + } + } + + for (auto &[name, v] : benches) { + std::sort(v.begin(), v.end()); + LOG(INFO) << "Bench " << name; + for (auto &[t, name] : v) { + LOG(INFO) << "\t" << name << " " << double(t) / 1000 << "s"; } } } -void test_dynamic_boc(BocOptions options) { - auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; - auto before = counter(); - SCOPE_EXIT { - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - << before << " vs " << counter(); - }; - td::Random::Xorshift128plus rnd{options.seed}; +DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { + DynamicBagOfCellsDb::Stats stats; + auto &rnd = options.rnd; std::string old_root_hash; std::string old_root_serialization; - auto kv = std::make_shared(); - auto create_dboc = [&]() { + DB db; + auto reload_db = [&]() { auto roots_n = old_root_hash.empty() ? 0 : 1; - return options.create_dboc(kv.get(), roots_n); + db = options.create_db(std::move(db), roots_n); }; - auto dboc = create_dboc(); - dboc->set_loader(std::make_unique(kv)); + reload_db(); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = create_dboc(); + reload_db(); } - dboc->set_loader(std::make_unique(kv)); + db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error(); + + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { - old_root = dboc->load_cell(old_root_hash).move_as_ok(); + old_root = db.dboc->load_cell(old_root_hash).move_as_ok(); auto serialization = serialize_boc(old_root); ASSERT_EQ(old_root_serialization, serialization); } @@ -1071,47 +1396,61 @@ void test_dynamic_boc(BocOptions options) { ->get_root_cell(0) .move_as_ok(); - dboc->dec(old_root); + db.dboc->dec(old_root); if (t != 0) { - dboc->inc(cell); - } - dboc->prepare_commit().ensure(); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer).ensure(); + db.dboc->inc(cell); } + options.commit(db, BocOptions::ResetCache); } - ASSERT_EQ(0u, kv->count("").ok()); + options.check_kv_is_empty(*db.kv); + + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc) { with_all_boc_options(test_dynamic_boc, 1); }; -void test_dynamic_boc2(BocOptions options) { - td::Random::Xorshift128plus rnd{options.seed}; +DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { + auto &rnd = options.rnd; + DynamicBagOfCellsDb::Stats stats; - int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); int max_roots = rnd.fast(1, 20); + int max_cells = 20; + + // VERBOSITY_NAME(boc) = 1; + // LOG(WARNING) << "====================================================\n\n"; + // max_roots = 2; + // total_roots = 2; + // max_cells = 2; + + auto meta_key = [](size_t i) { return PSTRING() << "meta." << i; }; + std::array meta; + int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; - auto kv = std::make_shared(); - auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; - auto dboc = create_dboc(0); - dboc->set_loader(std::make_unique(kv)); + DB db; + auto reload_db = [&](td::int64 root_n) { db = options.create_db(std::move(db), root_n); }; + reload_db(0); auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; auto before = counter(); - SCOPE_EXIT{ - // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - // << before << " vs " << counter(); + SCOPE_EXIT { + bool skip_check = false; + if (std::holds_alternative(options.options) && + std::get(options.options).use_arena) { + skip_check = true; + } + LOG_IF(FATAL, !(skip_check || before == counter())) << before << " vs " << counter(); }; std::vector> roots(max_roots); std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { - dboc->inc(root); + db.dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); roots[last_root_id % max_roots] = root; last_root_id++; @@ -1124,9 +1463,9 @@ void test_dynamic_boc2(BocOptions options) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; if (rnd() % 2 == 0) { - from_root = dboc->load_root(from_root_hash).move_as_ok(); + from_root = db.dboc->load_root(from_root_hash).move_as_ok(); } else { - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + from_root = db.dboc->load_cell(from_root_hash).move_as_ok(); } } else { VLOG(boc) << "FROM MEMORY"; @@ -1147,31 +1486,69 @@ void test_dynamic_boc2(BocOptions options) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); - root_cnt[new_root->get_hash()]++; - add_root(std::move(new_root)); + auto new_root_cell = gen_random_cell(rnd.fast(1, max_cells), from_root, rnd); + root_cnt[new_root_cell->get_hash()]++; + add_root(std::move(new_root_cell)); VLOG(boc) << " OK"; }; - auto commit = [&] { - VLOG(boc) << "commit"; - //rnd.fast(0, 1); - options.prepare_commit(*dboc); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + td::UsageStats commit_stats{}; + auto commit = [&](bool finish = false) { + for (size_t i = 0; i < meta.size(); i++) { + std::string value; + auto status = db.dboc->meta_get(meta_key(i), value).move_as_ok(); + if (status == KeyValue::GetStatus::Ok) { + ASSERT_EQ(value, meta[i]); + ASSERT_TRUE(!meta[i].empty()); + } else { + ASSERT_TRUE(meta[i].empty()); + } + + if (meta[i].empty()) { + if (!finish && rnd() % 2 == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } + } else { + auto f = finish ? 1 : rnd() % 3; + if (f == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } else if (f == 1) { + meta[i] = ""; + db.dboc->meta_erase(meta_key(i)); + VLOG(boc) << "meta erase " << meta_key(i); + } + } } - dboc->set_loader(std::make_unique(kv)); + + VLOG(boc) << "before commit cells_in_db=" << db.kv->count(""); + //rnd.fast(0, 1); + auto stats_before = db.kv->get_usage_stats(); + options.commit(db, BocOptions::ResetCache); + auto stats_after = db.kv->get_usage_stats(); + commit_stats = commit_stats + stats_after - stats_before; + VLOG(boc) << "after commit cells_in_db=" << db.kv->count(""); + + // db.reset_loader(); for (int i = last_commit_at; i < last_root_id; i++) { roots[i % max_roots].clear(); } last_commit_at = last_root_id; }; - auto reset = [&] { + auto reset = [&](bool force_full = false) { VLOG(boc) << "reset"; commit(); - dboc = create_dboc(td::int64(root_cnt.size())); - dboc->set_loader(std::make_unique(kv)); + if (rnd() % 3 == 0 || force_full) { + // very slow for rocksdb + auto r_stats = db.dboc->get_stats(); + if (r_stats.is_ok()) { + stats.apply_diff(r_stats.ok()); + } + reload_db(root_cnt.size()); + } }; auto delete_root = [&] { @@ -1187,22 +1564,28 @@ void test_dynamic_boc2(BocOptions options) { root_cnt.erase(it); } - dboc->dec(std::move(old_root)); + db.dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; td::RandomSteps steps({{new_root, 10}, {delete_root, 9}, {commit, 2}, {reset, 1}}); while (first_root_id != total_roots) { - VLOG(boc) << first_root_id << " " << last_root_id << " " << kv->count("").ok(); + VLOG(boc) << first_root_id << " " << last_root_id; // << " " << db.kv->count("").ok(); steps.step(rnd); } - commit(); - ASSERT_EQ(0u, kv->count("").ok()); + commit(true); + options.check_kv_is_empty(*db.kv); + + // auto stats = kv->get_usage_stats(); + // LOG(ERROR) << "total: " << stats; + reset(true); + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc2) { - with_all_boc_options(test_dynamic_boc2); + with_all_boc_options(test_dynamic_boc2, 50); } template @@ -1308,6 +1691,31 @@ TEST(TonDb, BocDeserializerSimpleThreads) { test_boc_deserializer_threads(); } +class RandomTree { + public: + RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) { + root_ = create(size); + } + Ref root() const { + return root_; + } + private: + Ref root_; + td::Random::Xorshift128plus rnd_; + Ref create(size_t size) { + CellBuilder cb; + cb.store_long(rnd_(), rnd_() % 63 + 1); + if (size > 0) { + td::uint64 rc = (rnd_() % 4) + 1; + for (td::uint64 i = 0; i < rc; i++) { + auto ref = create(size / rc); + cb.store_ref(std::move(ref)); + } + } + return cb.finalize(); + } +}; + class CompactArray { public: CompactArray(size_t size) { @@ -1341,6 +1749,10 @@ class CompactArray { size_t size() const { return size_; } + void reset() { + size_ = 0; + root_ = {}; + } Ref merkle_proof(std::vector keys) { std::set hashes; @@ -1435,6 +1847,282 @@ class FastCompactArray { std::vector v_; }; +struct BocTestHelper { + public: + BocTestHelper() = default; + BocTestHelper(td::int64 seed) : rnd_(seed) { + } + + CompactArray create_array(size_t size, td::uint64 max_value) { + std::vector v(size); + td::Random::Xorshift128plus rnd{123}; + for (auto &x : v) { + x = rnd() % max_value; + } + return CompactArray(v); + } + + private: + td::Random::Xorshift128plus rnd_{123}; +}; + +DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { + BocTestHelper helper(options.seed); + size_t n = 1 << 20; + size_t max_value = 1 << 26; + auto arr = helper.create_array(n, max_value); + + // auto kv = std::make_shared(); + td::Slice db_path = "compact_array_db"; + td::RocksDb::destroy(db_path).ensure(); + + DB db = options.create_db({}, {}); + DynamicBagOfCellsDb::Stats stats; + + td::Timer total_timer; + + auto bench = [&](td::Slice desc, auto &&f) { + auto before = db.dboc->get_stats().move_as_ok(); + td::Timer timer; + LOG(ERROR) << "Benchmarking " << desc; + f(); + stats.named_stats.stats_int[desc.str()] = td::int64(timer.elapsed() * 1000); + LOG(ERROR) << "Benchmarking " << desc << " done: " << timer.elapsed() << "s\n"; + auto after = db.dboc->get_stats().move_as_ok(); + after.named_stats.subtract_diff(before.named_stats); + LOG(ERROR) << after; + }; + + td::VectorQueue roots; + // Save array in db + bench(PSLICE() << "bench_inc_large_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + bench("bench_compactify", [&] { + auto status = dynamic_cast(*db.kv).raw_db()->CompactRange({}, nullptr, nullptr); + LOG_IF(FATAL, !status.ok()) << status.ToString(); + }); + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_inc_large_existed_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + + td::Random::Xorshift128plus rnd{123}; + while (false) { + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + td::Timer timer; + for (size_t i = 0; i < 10000; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + LOG(ERROR) << timer.elapsed() << "s\n"; + db.reset_loader(); + } + + for (auto p : std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + auto get_n = p.first; + auto set_n = p.second; + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + bench(PSTRING() << "bench_changes(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + for (size_t i = 0; i < get_n; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + for (size_t i = 0; i < set_n; i++) { + auto pos = rnd() % n; + auto value = rnd() % max_value; + arr.set(pos, value); + } + }); + bench(PSTRING() << "bench_commit(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + } + arr.reset(); + + bench(PSLICE() << "bench_dec_some_roots()", [&] { + while (roots.size() > 1) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + } + options.commit(db, BocOptions::ResetCache); + }); + + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_dec_large_root(n=" << n << ")", [&] { + while (!roots.empty()) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + + /* + do { + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + cell = {}; + options.prepare_commit(*db.dboc); + //db.dboc->prepare_commit().ensure(); + db.reset_loader(); + db = options.create_db(std::move(db), {}); + } while (true); + */ + } + options.commit(db, BocOptions::ResetCache); + }); + stats.named_stats.stats_int["bench_total"] = td::int64(total_timer.elapsed() * 1000); + + return stats; +} + +TEST(TonDb, BenchDynamicBocGetAndSet) { + with_all_boc_options(bench_dboc_get_and_set, 1); +} + +TEST(TonDb, DynamicBocIncSimple) { + auto kv = std::make_shared(std::make_shared()); + auto db = DynamicBagOfCellsDb::create_v2({.extra_threads = 0}); + db->set_loader(std::make_unique(kv)); + + td::Random::Xorshift128plus rnd(123); + size_t size = 4; + std::vector values(size); + for (auto &v : values) { + //v = rnd() % 2; + v = rnd(); + } + // 1. Create large dictionary and store it in db + auto arr_ptr = std::make_unique(values); + auto &arr = *arr_ptr; + td::VectorQueue queue; + auto push = [&]() { + //LOG(ERROR) << "PUSH ROOT"; + auto begin_stats = kv->get_usage_stats(); + db->inc(arr.root()); + queue.push(arr.root()->get_hash()); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + LOG(ERROR) << end_stats - begin_stats; + db->set_loader(std::make_unique(kv)); + auto hash = arr.root()->get_hash(); + arr = CompactArray{size, db->load_root(hash.as_slice()).move_as_ok()}; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto pop = [&]() { + if (queue.empty()) { + return; + } + //LOG(ERROR) << "POP ROOT"; + auto begin_stats = kv->get_usage_stats(); + auto cell = db->load_cell(queue.pop().as_slice()).move_as_ok(); + db->dec(cell); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + db->set_loader(std::make_unique(kv)); + //LOG(ERROR) << end_stats - begin_stats; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto upd = [&] { + for (int i = 0; i < 20; i++) { + auto pos = rnd.fast(0, td::narrow_cast(size) - 1); + if (rnd() % 2) { + auto value = rnd() % 2; + arr.set(pos, value); + } else { + arr.get(pos); + } + } + }; + + //LOG(ERROR) << "Created compact array"; + push(); + pop(); + //CHECK(kv->count("").move_as_ok() == 0); + + // 2. Lets change first 20 keys and read last 20 keys + /* + for (size_t i = 0; i < 20 && i < size; i++) { + arr.set(i, rnd()); + } + */ + //arr.set(0, rnd()); + arr.set(size - 1, rnd()); + for (size_t i = 0; i < 20 && i < size; i++) { + arr.get(size - i - 1); + } + + // 3. And now commit diff with stats + push(); + push(); + upd(); + upd(); + push(); + push(); + upd(); + pop(); + pop(); + upd(); + push(); + push(); + while (!queue.empty()) { + pop(); + } + LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); +} + +class BenchCellStorerMergeRefcntDiffs : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto left_update = CellStorer::serialize_refcnt_diffs(1); + auto right_update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_refcnt_diffs(left_update, right_update); + } + } + + private: + size_t tn_{}; +}; +class BenchCellStorerMergeValueAndRefcntDiff : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_value_and_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto value = CellStorer::serialize_value(10, cell, false); + auto update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_value_and_refcnt_diff(value, update); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, CellStorerMerge) { + bench(BenchCellStorerMergeRefcntDiffs()); + bench(BenchCellStorerMergeValueAndRefcntDiff()); +} + TEST(Cell, BocHands) { serialize_boc(CellBuilder{}.store_bytes("AAAAAAAA").finalize()); auto a = CellBuilder{}.store_bytes("abcd").store_ref(CellBuilder{}.store_bytes("???").finalize()).finalize(); @@ -2262,7 +2950,37 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { +TEST(UsageTree, ThreadSafe) { + size_t test_n = 100; + td::Random::Xorshift128plus rnd(123); + for (size_t test_i = 0; test_i < test_n; test_i++) { + auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + auto barrier = std::barrier{threads_n}; + std::vector threads; + std::vector explorations(threads_n); + for (std::ptrdiff_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, i = i]() { + barrier.arrive_and_wait(); + explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd); + }); + } + for (auto &thread : threads) { + thread.join(); + } + auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); + auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + for (auto &exploration : explorations) { + auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); + ASSERT_EQ(exploration.log, new_exploration.log); + } + } +} + +/* +vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); @@ -2283,19 +3001,19 @@ void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); auto serialization = serialize_boc(cell); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); + vm::DynamicBagOfCellsDb::Stats stats; + return stats; } TEST(TonDb, DynamicBocRespectsUsageCell) { - vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); + vm::with_all_boc_options(test_dynamic_boc_respects_usage_cell, 20, true); } +*/ TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; - size_t n = 1000000; - std::vector data(n); - std::iota(data.begin(), data.end(), 0); - vm::CompactArray arr(data); - auto root = arr.root(); + vm::RandomTree tree(100000, rnd); + auto root = tree.root(); std::string path = "serialization"; td::unlink(path).ignore(); auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) @@ -2313,12 +3031,20 @@ TEST(TonDb, LargeBocSerializer) { dboc->commit(cell_storer); dboc->set_loader(std::make_unique(kv)); td::unlink(path).ignore(); - fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) - .move_as_ok(); - std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); - fd.close(); - auto b = td::read_file_str(path).move_as_ok(); - CHECK(a == b); + td::string prev_b(""); + auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); + for (int i = 0; i < 4; i++) { + fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + fd.close(); + auto b = td::read_file_str(path).move_as_ok(); + + auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); + ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); + if (i > 0) ASSERT_EQ(prev_b, b); + prev_b = b; + } } TEST(TonDb, DoNotMakeListsPrunned) { diff --git a/crypto/test/test-ed25519-crypto.cpp b/crypto/test/test-ed25519-crypto.cpp deleted file mode 100644 index 3e3dab896..000000000 --- a/crypto/test/test-ed25519-crypto.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include -#include -#include -#include - -#include "crypto/ellcurve/Ed25519.h" - -static void my_assert_impl(bool cond, const char* str, const char* file, int line) { - if (!cond) { - std::cerr << "Failed " << str << " in " << file << " at " << line << ".\n"; - } -} -#define my_assert(x) my_assert_impl(x, #x, __FILE__, __LINE__) - -void print_buffer(const unsigned char buffer[32]) { - for (int i = 0; i < 32; i++) { - char buff[4]; - sprintf(buff, "%02x", buffer[i]); - std::cout << buff; - } -} - -std::string buffer_to_hex(const unsigned char* buffer, std::size_t size = 32) { - const char* hex = "0123456789ABCDEF"; - std::string res(2 * size, '\0'); - for (std::size_t i = 0; i < size; i++) { - auto c = buffer[i]; - res[2 * i] = hex[c & 15]; - res[2 * i + 1] = hex[c >> 4]; - } - return res; -} - -// export of (17/12)G on twisted Edwards curve -unsigned char test_vector1[32] = {0xfc, 0xb7, 0x42, 0x1e, 0x26, 0xad, 0x1b, 0x17, 0xf6, 0xb1, 0x52, - 0x0c, 0xdb, 0x8a, 0x64, 0x7d, 0x28, 0xa7, 0x56, 0x69, 0xd4, 0xb6, - 0x0c, 0xec, 0x63, 0x72, 0x5e, 0xe6, 0x32, 0x4d, 0xf7, 0xe6}; - -unsigned char rfc7748_output[32] = { - 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, - 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57, -}; - -bool test_ed25519_impl(void) { - std::cout << "************** Testing Curve25519 / Ed25519 operations ************\n"; - auto& E = ellcurve::Curve25519(); - auto& Edw = ellcurve::Ed25519(); - arith::Bignum L = E.get_ell(); - my_assert(arith::is_prime(L)); - my_assert(L == Edw.get_ell()); - arith::ResidueRing Fl(L); - arith::Bignum s = Fl.frac(17, 12).extract(); - arith::Bignum t = Fl.frac(12, 17).extract(); - std::cout << "l = " << L << std::endl; - std::cout << "s = 17/12 mod l = " << s << std::endl; - std::cout << "t = 12/17 mod l = " << t << std::endl; - auto sG = E.power_gen_xz(s); - auto u_sG = sG.get_u(); - std::cout << "Curve25519 u(sG) = " << sG.get_u().extract() << std::endl; - std::cout << "Curve25519 y(sG) = " << sG.get_y().extract() << std::endl; - auto sG1 = Edw.power_gen(s); - std::cout << "Ed25519 u(sG) = " << sG1.get_u().extract() << std::endl; - std::cout << "Ed25519 y(sG) = " << sG1.get_y().extract() << std::endl; - std::cout << "Ed25519 x(sG) = " << sG1.get_x().extract() << std::endl; - my_assert(sG1.get_x().extract() != sG1.get_y().extract()); - my_assert(sG.get_u() == sG1.get_u()); - my_assert(sG.get_y() == sG1.get_y()); - - my_assert( - sG1.get_x().extract() == - arith::Bignum(arith::dec_string{"9227429025021714590777223519505276506601225973596506606120015751301699519597"})); - my_assert(sG1.get_y().extract() == - arith::Bignum( - arith::dec_string{"46572854587220149033453000581008590225032365765275643343836649812808016508924"})); - - auto sG2 = Edw.power_gen(s, true); - my_assert(sG1.get_u() == sG2.get_u()); - my_assert(sG1.get_y() == sG2.get_y()); - unsigned char buff[32]; - std::memset(buff, 0, 32); - my_assert(sG1.export_point(buff)); - std::cout << "sG export = " << buffer_to_hex(buff) << std::endl; - bool ok; - auto sG3 = Edw.import_point(buff, ok); - my_assert(ok); - my_assert(!std::memcmp(buff, test_vector1, 32)); - my_assert(sG3.get_u() == sG1.get_u()); - my_assert(sG2.get_x() == sG2.get_x()); - my_assert(sG2.get_y() == sG2.get_y()); - - auto stG = E.power_xz(u_sG, t); - std::cout << "Curve25519 u(stG) = " << stG.get_u().extract() << std::endl; - my_assert(stG.get_u().extract() == 9); - auto stG1 = Edw.power_point(sG1, t); - std::cout << "Ed25519 u(stG) = " << stG1.get_u().extract() << std::endl; - my_assert(stG1.get_u().extract() == 9); - stG1.normalize(); - my_assert(stG1.XY == Edw.get_base_point().XY); - my_assert(stG1.X == Edw.get_base_point().X); - my_assert(stG1.Y == Edw.get_base_point().Y); - my_assert(stG1.Z == Edw.get_base_point().Z); - auto stG2 = Edw.power_point(sG2, t, true); - my_assert(stG2.get_u().extract() == 9); - stG2.normalize(); - my_assert(stG2.XY == stG1.XY && stG2.X == stG1.X && stG2.Y == stG1.Y); - auto stG3 = Edw.power_point(sG3, t).normalize(); - auto stG4 = Edw.power_point(sG3, t, true).normalize(); - my_assert(stG3.XY == stG1.XY && stG3.X == stG1.X && stG3.Y == stG1.Y); - my_assert(stG4.XY == stG1.XY && stG4.X == stG1.X && stG4.Y == stG1.Y); - - // RFC7748 test vector - auto u = - arith::Bignum(arith::dec_string{"8883857351183929894090759386610649319417338800022198945255395922347792736741"}); - //u[255] = 0; - auto n = - arith::Bignum(arith::dec_string{"35156891815674817266734212754503633747128614016119564763269015315466259359304"}); - //n[255] = 0; n[254] = 1; - //n[0] = n[1] = n[2] = 0; - auto umodp = arith::Residue(u, E.get_base_ring()); - auto nP = E.power_xz(umodp, n); - std::cout << "u(P) = " << u.to_hex() << std::endl; - std::cout << "n = " << n.to_hex() << std::endl; - std::cout << "u(nP) = " << nP.get_u().extract().to_hex() << std::endl; - unsigned char buffer[32]; - std::memset(buffer, 0, 32); - nP.export_point_u(buffer); - std::cout << "u(nP) export = " << buffer_to_hex(buffer) << std::endl; - my_assert(!std::memcmp(buffer, rfc7748_output, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -unsigned char fixed_privkey[32] = "abacabadabacabaeabacabadabacaba"; -unsigned char fixed_pubkey[32] = {0x6f, 0x9e, 0x5b, 0xde, 0xce, 0x87, 0x21, 0xeb, 0x57, 0x37, 0xfb, - 0xb5, 0x92, 0x28, 0xba, 0x07, 0xf7, 0x88, 0x0f, 0x73, 0xce, 0x5b, - 0xfa, 0xa1, 0xb7, 0x15, 0x73, 0x03, 0xd4, 0x20, 0x1e, 0x74}; - -unsigned char rfc8032_secret_key1[32] = {0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, - 0xf4, 0x92, 0xec, 0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, - 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, 0x1c, 0xae, 0x7f, 0x60}; - -unsigned char rfc8032_public_key1[32] = {0xd7, 0x5a, 0x98, 0x01, 0x82, 0xb1, 0x0a, 0xb7, 0xd5, 0x4b, 0xfe, - 0xd3, 0xc9, 0x64, 0x07, 0x3a, 0x0e, 0xe1, 0x72, 0xf3, 0xda, 0xa6, - 0x23, 0x25, 0xaf, 0x02, 0x1a, 0x68, 0xf7, 0x07, 0x51, 0x1a}; - -unsigned char rfc8032_signature1[64] = { - 0xe5, 0x56, 0x43, 0x00, 0xc3, 0x60, 0xac, 0x72, 0x90, 0x86, 0xe2, 0xcc, 0x80, 0x6e, 0x82, 0x8a, - 0x84, 0x87, 0x7f, 0x1e, 0xb8, 0xe5, 0xd9, 0x74, 0xd8, 0x73, 0xe0, 0x65, 0x22, 0x49, 0x01, 0x55, - 0x5f, 0xb8, 0x82, 0x15, 0x90, 0xa3, 0x3b, 0xac, 0xc6, 0x1e, 0x39, 0x70, 0x1c, 0xf9, 0xb4, 0x6b, - 0xd2, 0x5b, 0xf5, 0xf0, 0x59, 0x5b, 0xbe, 0x24, 0x65, 0x51, 0x41, 0x43, 0x8e, 0x7a, 0x10, 0x0b, -}; - -unsigned char rfc8032_secret_key2[32] = { - 0xc5, 0xaa, 0x8d, 0xf4, 0x3f, 0x9f, 0x83, 0x7b, 0xed, 0xb7, 0x44, 0x2f, 0x31, 0xdc, 0xb7, 0xb1, - 0x66, 0xd3, 0x85, 0x35, 0x07, 0x6f, 0x09, 0x4b, 0x85, 0xce, 0x3a, 0x2e, 0x0b, 0x44, 0x58, 0xf7, -}; - -unsigned char rfc8032_public_key2[32] = { - 0xfc, 0x51, 0xcd, 0x8e, 0x62, 0x18, 0xa1, 0xa3, 0x8d, 0xa4, 0x7e, 0xd0, 0x02, 0x30, 0xf0, 0x58, - 0x08, 0x16, 0xed, 0x13, 0xba, 0x33, 0x03, 0xac, 0x5d, 0xeb, 0x91, 0x15, 0x48, 0x90, 0x80, 0x25, -}; - -unsigned char rfc8032_message2[2] = {0xaf, 0x82}; - -unsigned char rfc8032_signature2[64] = { - 0x62, 0x91, 0xd6, 0x57, 0xde, 0xec, 0x24, 0x02, 0x48, 0x27, 0xe6, 0x9c, 0x3a, 0xbe, 0x01, 0xa3, - 0x0c, 0xe5, 0x48, 0xa2, 0x84, 0x74, 0x3a, 0x44, 0x5e, 0x36, 0x80, 0xd7, 0xdb, 0x5a, 0xc3, 0xac, - 0x18, 0xff, 0x9b, 0x53, 0x8d, 0x16, 0xf2, 0x90, 0xae, 0x67, 0xf7, 0x60, 0x98, 0x4d, 0xc6, 0x59, - 0x4a, 0x7c, 0x15, 0xe9, 0x71, 0x6e, 0xd2, 0x8d, 0xc0, 0x27, 0xbe, 0xce, 0xea, 0x1e, 0xc4, 0x0a, -}; - -bool test_ed25519_crypto() { - std::cout << "************** Testing Curve25519 / Ed25519 cryptographic primitives ************\n"; - crypto::Ed25519::PrivateKey PK1, PK2, PK3, PK4, PK5; - PK1.random_private_key(); - PK2.import_private_key(fixed_privkey); - unsigned char priv2_export[32]; - bool ok = PK1.export_private_key(priv2_export); - std::cout << "PK1 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - ok = PK2.export_private_key(priv2_export); - std::cout << "PK2 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - PK3.import_private_key(priv2_export); - std::cout << "PK3 = " << PK3.ok() << std::endl; - my_assert(PK3.ok()); - - unsigned char pub_export[32]; - ok = PK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - crypto::Ed25519::PublicKey PubK1(pub_export); - ok = PK2.export_public_key(pub_export); - std::cout << "PubK2 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK2(pub_export); - ok = PK3.export_public_key(pub_export); - std::cout << "PubK3 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK3(pub_export); - ok = PubK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - - unsigned char secret22[32]; - ok = PK2.compute_shared_secret(secret22, PubK3); - std::cout << "secret(PK2,PubK2)=" << ok << " " << buffer_to_hex(secret22) << std::endl; - my_assert(ok); - - unsigned char secret12[32], secret21[32]; - ok = PK1.compute_shared_secret(secret12, PubK3); - std::cout << "secret(PK1,PubK2)=" << ok << " " << buffer_to_hex(secret12) << std::endl; - my_assert(ok); - ok = PK2.compute_shared_secret(secret21, PubK1); - std::cout << "secret(PK2,PubK1)=" << ok << " " << buffer_to_hex(secret21) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(secret12, secret21, 32)); - - // for (int i = 0; i < 1000; i++) { - // ok = PK1.compute_shared_secret(secret12, PubK3); - // my_assert(ok); - // ok = PK2.compute_shared_secret(secret21, PubK1); - // my_assert(ok); - // } - - unsigned char signature[64]; - ok = PK1.sign_message(signature, (const unsigned char*)"abc", 3); - std::cout << "PK1.signature=" << ok << " " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - - // signature[63] ^= 1; - ok = PubK1.check_message_signature(signature, (const unsigned char*)"abc", 3); - std::cout << "PubK1.check_signature=" << ok << std::endl; - my_assert(ok); - - PK4.import_private_key(rfc8032_secret_key1); - PK4.export_public_key(pub_export); - std::cout << "PK4.private_key = " << buffer_to_hex(rfc8032_secret_key1) << std::endl; - std::cout << "PK4.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key1, 32)); - ok = PK4.sign_message(signature, (const unsigned char*)"", 0); - std::cout << "PK4.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature1, 32)); - - PK5.import_private_key(rfc8032_secret_key2); - PK5.export_public_key(pub_export); - std::cout << "PK5.private_key = " << buffer_to_hex(rfc8032_secret_key2) << std::endl; - std::cout << "PK5.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key2, 32)); - ok = PK5.sign_message(signature, rfc8032_message2, 2); - std::cout << "PK5.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature2, 32)); - crypto::Ed25519::PublicKey PubK5(pub_export); - - // for (int i = 0; i < 10000; i++) { - // ok = PK5.sign_message (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - // for (int i = 0; i < 10000; i++) { - // ok = PubK5.check_message_signature (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - - unsigned char temp_pubkey[32]; - crypto::Ed25519::TempKeyGenerator TKG; // use one generator a lot of times - - TKG.create_temp_shared_secret(temp_pubkey, secret12, PubK1, (const unsigned char*)"abc", 3); - std::cout << "secret12=" << buffer_to_hex(secret12) << "; temp_pubkey=" << buffer_to_hex(temp_pubkey) << std::endl; - - PK1.compute_temp_shared_secret(secret21, temp_pubkey); - std::cout << "secret21=" << buffer_to_hex(secret21) << std::endl; - my_assert(!std::memcmp(secret12, secret21, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -int main(void) { - test_ed25519_impl(); - test_ed25519_crypto(); - return 0; -} diff --git a/crypto/util/Miner.cpp b/crypto/util/Miner.cpp deleted file mode 100644 index 3e26fac6b..000000000 --- a/crypto/util/Miner.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Miner.h" - -#include "td/utils/Random.h" -#include "td/utils/misc.h" -#include "td/utils/crypto.h" -#include "td/utils/port/Clocks.h" -#include - -namespace ton { -#pragma pack(push, 1) -struct HData { - unsigned char op[4]; - signed char flags = -4; - unsigned char expire[4] = {}, myaddr[32] = {}, rdata1[32] = {}, pseed[16] = {}, rdata2[32] = {}; - void inc() { - for (long i = 31; !(rdata1[i] = ++(rdata2[i])); --i) { - } - } - void set_expire(unsigned x) { - for (int i = 3; i >= 0; --i) { - expire[i] = (x & 0xff); - x >>= 8; - } - } - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } -}; - -struct HDataEnv { - unsigned char d1 = 0, d2 = sizeof(HData) * 2; - HData body; - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } - - void init(const block::StdAddress& my_address, td::Slice seed) { - std::memcpy(body.myaddr, my_address.addr.data(), sizeof(body.myaddr)); - body.flags = static_cast(my_address.workchain * 4 + my_address.bounceable); - CHECK(seed.size() == 16); - std::memcpy(body.pseed, seed.data(), 16); - std::memcpy(body.op, "Mine", 4); - - td::Random::secure_bytes(body.rdata1, 32); - std::memcpy(body.rdata2, body.rdata1, 32); - } -}; - -static_assert(std::is_trivially_copyable::value, "HDataEnv must be a trivial type"); -#pragma pack(pop) - -td::optional Miner::run(const Options& options) { - HDataEnv H; - H.init(options.my_address, td::Slice(options.seed.data(), options.seed.size())); - - td::Slice data = H.as_slice(); - CHECK(data.size() == 123); - - constexpr size_t prefix_size = 72; - constexpr size_t guard_pos = prefix_size - (72 - 28); - CHECK(0 <= guard_pos && guard_pos < 32); - size_t got_prefix_size = (const unsigned char*)H.body.rdata1 + guard_pos + 1 - (const unsigned char*)&H; - CHECK(prefix_size == got_prefix_size); - - auto head = data.substr(0, prefix_size); - auto tail = data.substr(prefix_size); - - SHA256_CTX shactx1, shactx2; - std::array hash; - SHA256_Init(&shactx1); - auto guard = head.back(); - - td::int64 i = 0, i0 = 0; - for (; i < options.max_iterations; i++) { - if (!(i & 0xfffff) || head.back() != guard) { - if (options.token_) { - break; - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - i0 = i; - if (options.expire_at && options.expire_at.value().is_in_past(td::Timestamp::now())) { - break; - } - H.body.set_expire((unsigned)td::Clocks::system() + 900); - guard = head.back(); - SHA256_Init(&shactx1); - SHA256_Update(&shactx1, head.ubegin(), head.size()); - } - shactx2 = shactx1; - SHA256_Update(&shactx2, tail.ubegin(), tail.size()); - SHA256_Final(hash.data(), &shactx2); - - if (memcmp(hash.data(), options.complexity.data(), 32) < 0) { - // FOUND - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return H.body.as_slice().str(); - } - H.body.inc(); - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return {}; -} -} // namespace ton diff --git a/crypto/util/Miner.h b/crypto/util/Miner.h deleted file mode 100644 index f91a66866..000000000 --- a/crypto/util/Miner.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "block/block.h" -#include "td/utils/CancellationToken.h" -#include "td/utils/optional.h" -#include "td/utils/Time.h" -#include -#include - -namespace ton { -class Miner { - public: - struct Options { - block::StdAddress my_address; - std::array seed; - std::array complexity; - td::optional expire_at; - td::int64 max_iterations = std::numeric_limits::max(); - std::atomic* hashes_computed{nullptr}; - td::CancellationToken token_; - }; - - static td::optional run(const Options& options); -}; -} // namespace ton diff --git a/crypto/util/pow-miner.cpp b/crypto/util/pow-miner.cpp deleted file mode 100644 index c065fdc70..000000000 --- a/crypto/util/pow-miner.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "common/bigint.hpp" -#include "common/refint.h" -#include "block/block.h" -#include "td/utils/benchmark.h" -#include "td/utils/filesystem.h" -#include "vm/boc.h" -#include "openssl/digest.hpp" -#include -#include -#include -#include -#include -#include "git.h" -#include "Miner.h" - -const char* progname; - -int usage() { - std::cerr - << "usage: " << progname - << " [-v][-B][-w] [-t] [ " - "] [-V]\n" - "Outputs a valid value for proof-of-work testgiver after computing at most hashes " - "or terminates with non-zero exit code\n"; - std::exit(2); -} - -td::RefInt256 parse_bigint(std::string str, int bits) { - int len = (int)str.size(); - auto num = td::make_refint(); - auto& x = num.write(); - if (len >= 3 && str[0] == '0' && str[1] == 'x') { - if (x.parse_hex(str.data() + 2, len - 2) != len - 2) { - return {}; - } - } else if (!len || x.parse_dec(str.data(), len) != len) { - return {}; - } - return x.unsigned_fits_bits(bits) ? std::move(num) : td::RefInt256{}; -} - -td::RefInt256 parse_bigint_chk(std::string str, int bits) { - auto x = parse_bigint(std::move(str), bits); - if (x.is_null()) { - std::cerr << "fatal: `" << str << "` is not an integer" << std::endl; - usage(); - } - return x; -} - -void parse_addr(std::string str, block::StdAddress& addr) { - if (!addr.parse_addr(str) || (addr.workchain != -1 && addr.workchain != 0)) { - std::cerr << "fatal: `" << str.c_str() << "` is not a valid blockchain address" << std::endl; - usage(); - } -} - -bool make_boc = false; -std::string boc_filename; -block::StdAddress miner_address; - -int verbosity = 0; -std::atomic hashes_computed{0}; -td::Timestamp start_at; - -void print_stats() { - auto passed = td::Timestamp::now().at() - start_at.at(); - if (passed < 1e-9) { - passed = 1; - } - std::cerr << "[ hashes computed: " << hashes_computed << " ]" << std::endl; - std::cerr << "[ speed: " << static_cast(hashes_computed) / passed << " hps ]" << std::endl; -} - -int found(td::Slice data) { - for (unsigned i = 0; i < data.size(); i++) { - printf("%02X", data.ubegin()[i]); - } - printf("\n"); - if (make_boc) { - vm::CellBuilder cb; - td::Ref ext_msg, body; - CHECK(cb.store_bytes_bool(data) // body - && cb.finalize_to(body) // -> body - && cb.store_long_bool(0x44, 7) // ext_message_in$10 ... - && cb.store_long_bool(miner_address.workchain, 8) // workchain - && cb.store_bytes_bool(miner_address.addr.as_slice()) // miner addr - && cb.store_long_bool(1, 6) // amount:Grams ... - && cb.store_ref_bool(std::move(body)) // body:^Cell - && cb.finalize_to(ext_msg)); - auto boc = vm::std_boc_serialize(std::move(ext_msg), 2).move_as_ok(); - std::cerr << "Saving " << boc.size() << " bytes of serialized external message into file `" << boc_filename << "`" - << std::endl; - td::write_file(boc_filename, boc).ensure(); - } - if (verbosity > 0) { - print_stats(); - } - std::exit(0); - return 0; -} - -void miner(const ton::Miner::Options& options) { - auto res = ton::Miner::run(options); - if (res) { - found(res.value()); - } -} - -class MinerBench : public td::Benchmark { - public: - std::string get_description() const override { - return "Miner"; - } - - void run(int n) override { - ton::Miner::Options options; - options.my_address.parse_addr("EQDU86V5wyPrLd4nQ0RHPcCLPZq_y1O5wFWyTsMw63vjXTOv"); - std::fill(options.seed.begin(), options.seed.end(), 0xa7); - std::fill(options.complexity.begin(), options.complexity.end(), 0); - options.max_iterations = n; - CHECK(!ton::Miner::run(options)); - } -}; - -int main(int argc, char* const argv[]) { - ton::Miner::Options options; - - progname = argv[0]; - int i, threads = 0; - bool bounce = false, benchmark = false; - while ((i = getopt(argc, argv, "bnvw:t:Bh:V")) != -1) { - switch (i) { - case 'v': - ++verbosity; - break; - case 'w': - threads = atoi(optarg); - CHECK(threads > 0 && threads <= 256); - break; - case 't': { - int timeout = atoi(optarg); - CHECK(timeout > 0); - options.expire_at = td::Timestamp::in(timeout); - break; - } - case 'B': - benchmark = true; - break; - case 'b': - bounce = true; - break; - case 'n': - bounce = false; - break; - case 'V': - std::cout << "pow-miner build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - std::exit(0); - break; - case 'h': - return usage(); - default: - std::cerr << "unknown option" << std::endl; - return usage(); - } - } - if (benchmark && argc == optind) { - td::bench(MinerBench()); - return 0; - } - - if (argc != optind + 4 && argc != optind + 6) { - return usage(); - } - - parse_addr(argv[optind], options.my_address); - options.my_address.bounceable = bounce; - CHECK(parse_bigint_chk(argv[optind + 1], 128)->export_bytes(options.seed.data(), 16, false)); - - auto cmplx = parse_bigint_chk(argv[optind + 2], 256); - CHECK(cmplx->export_bytes(options.complexity.data(), 32, false)); - CHECK(!cmplx->unsigned_fits_bits(256 - 62)); - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*cmplx, hrate); - long long hash_rate = hrate.to_long(); - options.max_iterations = parse_bigint_chk(argv[optind + 3], 50)->to_long(); - if (argc == optind + 6) { - make_boc = true; - parse_addr(argv[optind + 4], miner_address); - boc_filename = argv[optind + 5]; - } - - if (verbosity >= 2) { - std::cerr << "[ expected required hashes for success: " << hash_rate << " ]" << std::endl; - } - if (benchmark) { - td::bench(MinerBench()); - } - - start_at = td::Timestamp::now(); - - options.hashes_computed = &hashes_computed; - // may invoke several miner threads - if (threads <= 0) { - miner(options); - } else { - std::vector T; - for (int i = 0; i < threads; i++) { - T.emplace_back(miner, options); - } - for (auto& thr : T) { - thr.join(); - } - } - if (verbosity > 0) { - print_stats(); - } -} diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp new file mode 100644 index 000000000..d37e0380a --- /dev/null +++ b/crypto/vm/boc-compression.cpp @@ -0,0 +1,625 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "boc-compression.h" + +#include +#include +#include "vm/boc.h" +#include "vm/boc-writers.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/lz4.h" + +namespace vm { + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots) { + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(boc_roots), 2)); + td::BufferSlice compressed = td::lz4_compress(data); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(data.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size) { + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + int decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size <= 0 || decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + return roots; +} + +inline void append_uint(td::BitString& bs, unsigned val, unsigned n) { + bs.reserve_bitslice(n).bits().store_uint(val, n); +} + +inline td::Result read_uint(td::BitSlice& bs, int bits) { + // Check if there enough bits available + if (bs.size() < bits) { + return td::Status::Error("BOC decompression failed: not enough bits to read"); + } + unsigned result = bs.bits().get_uint(bits); + bs.advance(bits); + return result; +} + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots) { + // Input validation + if (boc_roots.empty()) { + return td::Status::Error("No root cells were provided for serialization"); + } + for (const auto& root : boc_roots) { + if (root.is_null()) { + return td::Status::Error("Cannot serialize a null cell reference into a bag of cells"); + } + } + + // Initialize data structures for graph representation + td::HashMap cell_hashes; + std::vector> boc_graph; + std::vector refs_cnt; + std::vector cell_data; + std::vector cell_type; + std::vector prunned_branch_level; + std::vector root_indexes; + size_t total_size_estimate = 0; + + // Build graph representation using recursive lambda + const auto build_graph = [&](auto&& self, td::Ref cell) -> td::Result { + if (cell.is_null()) { + return td::Status::Error("Error while importing a cell during serialization: cell is null"); + } + + auto cell_hash = cell->get_hash(); + auto it = cell_hashes.find(cell_hash); + if (it != cell_hashes.end()) { + return it->second; + } + + size_t current_cell_id = boc_graph.size(); + cell_hashes.emplace(cell_hash, current_cell_id); + + bool is_special = false; + vm::CellSlice cell_slice = vm::load_cell_slice_special(cell, is_special); + if (!cell_slice.is_valid()) { + return td::Status::Error("Invalid loaded cell data"); + } + td::BitSlice cell_bitslice = cell_slice.as_bitslice(); + + // Initialize new cell in graph + boc_graph.emplace_back(); + refs_cnt.emplace_back(cell_slice.size_refs()); + cell_type.emplace_back(size_t(cell_slice.special_type())); + prunned_branch_level.push_back(0); + + DCHECK(cell_slice.size_refs() <= 4); + + // Process special cell of type PrunnedBranch + if (cell_slice.special_type() == vm::CellTraits::SpecialType::PrunnedBranch) { + DCHECK(cell_slice.size() >= 16); + cell_data.emplace_back(cell_bitslice.subslice(16, cell_bitslice.size() - 16)); + prunned_branch_level.back() = cell_slice.data()[1]; + } else { + cell_data.emplace_back(cell_bitslice); + } + total_size_estimate += cell_bitslice.size(); + + // Process cell references + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); + boc_graph[current_cell_id][i] = child_id; + } + + return current_cell_id; + }; + + // Build the graph starting from roots + for (auto root : boc_roots) { + TRY_RESULT(root_cell_id, build_graph(build_graph, root)); + root_indexes.push_back(root_cell_id); + } + + // Check graph properties + const size_t node_count = boc_graph.size(); + std::vector> reverse_graph(node_count); + size_t edge_count = 0; + + // Build reverse graph + for (int i = 0; i < node_count; ++i) { + for (size_t child_index = 0; child_index < refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + ++edge_count; + reverse_graph[child].push_back(i); + } + } + + // Process cell data sizes + std::vector is_data_small(node_count, 0); + for (int i = 0; i < node_count; ++i) { + if (cell_type[i] != 1) { + is_data_small[i] = cell_data[i].size() < 128; + } + } + + // Perform topological sort + std::vector topo_order, rank(node_count); + const auto topological_sort = [&]() -> td::Status { + std::vector> queue; + queue.reserve(node_count); + std::vector in_degree(node_count); + + // Calculate in-degrees and initialize queue + for (int i = 0; i < node_count; ++i) { + in_degree[i] = refs_cnt[i]; + if (in_degree[i] == 0) { + queue.emplace_back(cell_type[i] == 0, -int(cell_data[i].size()), -i); + } + } + + if (queue.empty()) { + return td::Status::Error("Cycle detected in cell references"); + } + + std::sort(queue.begin(), queue.end()); + + // Process queue + while (!queue.empty()) { + int node = -std::get<2>(queue.back()); + queue.pop_back(); + topo_order.push_back(node); + + for (int parent : reverse_graph[node]) { + if (--in_degree[parent] == 0) { + queue.emplace_back(0, 0, -parent); + } + } + } + + if (topo_order.size() != node_count) { + return td::Status::Error("Invalid graph structure"); + } + + std::reverse(topo_order.begin(), topo_order.end()); + return td::Status::OK(); + }; + + TRY_STATUS(topological_sort()); + + // Calculate index of vertices in topsort + for (int i = 0; i < node_count; ++i) { + rank[topo_order[i]] = i; + } + + // Build compressed representation + td::BitString result; + total_size_estimate += (node_count * 10 * 8); + result.reserve_bits(total_size_estimate); + + // Store roots information + append_uint(result, root_indexes.size(), 32); + for (int root_ind : root_indexes) { + append_uint(result, rank[root_ind], 32); + } + + // Store node count + append_uint(result, node_count, 32); + + // Store cell types and sizes + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + size_t currrent_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; + append_uint(result, currrent_cell_type, 4); + append_uint(result, refs_cnt[node], 4); + + if (cell_type[node] != 1) { + if (is_data_small[node]) { + append_uint(result, 1, 1); + append_uint(result, cell_data[node].size(), 7); + } else { + append_uint(result, 0, 1); + append_uint(result, 1 + cell_data[node].size() / 8, 7); + } + } + } + + // Store edge information + auto edge_bits = result.reserve_bitslice(edge_count).bits(); + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + for (size_t child_index = 0; child_index < refs_cnt[node]; ++child_index) { + size_t child = boc_graph[node][child_index]; + edge_bits.store_uint(rank[child] == i + 1, 1); + ++edge_bits; + } + } + + // Store cell data + for (size_t node : topo_order) { + if (cell_type[node] != 1 && !is_data_small[node]) { + continue; + } + result.append(cell_data[node].subslice(0, cell_data[node].size() % 8)); + } + + // Store BOC graph with optimized encoding + for (size_t i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + if (node_count <= i + 3) + continue; + + for (int j = 0; j < refs_cnt[node]; ++j) { + if (rank[boc_graph[node][j]] <= i + 1) + continue; + + int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); + + if (required_bits < 8 - (result.size() + 1) % 8 + 1) { + append_uint(result, delta, required_bits); + } else if (delta < (1 << (8 - (result.size() + 1) % 8))) { + size_t available_bits = 8 - (result.size() + 1) % 8; + append_uint(result, 1, 1); + append_uint(result, delta, available_bits); + } else { + append_uint(result, 0, 1); + append_uint(result, delta, required_bits); + } + } + } + + // Pad result to byte boundary + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Store remaining cell data + for (size_t node : topo_order) { + if (cell_type[node] == 1 || is_data_small[node]) { + size_t prefix_size = cell_data[node].size() % 8; + result.append(cell_data[node].subslice(prefix_size, cell_data[node].size() - prefix_size)); + } else { + size_t data_size = cell_data[node].size() + 1; + size_t padding = (8 - data_size % 8) % 8; + + if (padding) { + append_uint(result, 0, padding); + } + append_uint(result, 1, 1); + result.append(cell_data[node]); + } + } + + // Final padding + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Create final compressed buffer + td::BufferSlice serialized((const char*)result.bits().get_byte_ptr(), result.size() / 8); + + td::BufferSlice compressed = td::lz4_compress(serialized); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(serialized.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { + constexpr size_t kMaxCellDataLengthBits = 1024; + + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + size_t decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + // Decompress LZ4 data + TRY_RESULT(serialized, td::lz4_decompress(compressed, decompressed_size)); + + if (serialized.size() != decompressed_size) { + return td::Status::Error("BOC decompression failed: decompressed size mismatch"); + } + + // Initialize bit reader + td::BitSlice bit_reader(serialized.as_slice().ubegin(), serialized.as_slice().size() * 8); + size_t orig_size = bit_reader.size(); + + // Read root count + TRY_RESULT(root_count, read_uint(bit_reader, 32)); + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided root_count is incorrect + if (root_count < 1 || root_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid root count"); + } + + std::vector root_indexes(root_count); + for (int i = 0; i < root_count; ++i) { + TRY_RESULT_ASSIGN(root_indexes[i], read_uint(bit_reader, 32)); + } + + // Read number of nodes from header + TRY_RESULT(node_count, read_uint(bit_reader, 32)); + if (node_count < 1) { + return td::Status::Error("BOC decompression failed: invalid node count"); + } + + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided node_count is incorrect + if (node_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: incorrect node count provided"); + } + + + // Validate root indexes + for (int i = 0; i < root_count; ++i) { + if (root_indexes[i] >= node_count) { + return td::Status::Error("BOC decompression failed: invalid root index"); + } + } + + // Initialize data structures + std::vector cell_data_length(node_count), is_data_small(node_count); + std::vector is_special(node_count), cell_refs_cnt(node_count); + std::vector prunned_branch_level(node_count, 0); + + std::vector cell_builders(node_count); + std::vector> boc_graph(node_count); + + // Read cell metadata + for (int i = 0; i < node_count; ++i) { + // Check enough bits for cell type and refs count + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for cell metadata"); + } + + size_t cell_type = bit_reader.bits().get_uint(4); + is_special[i] = bool(cell_type); + if (is_special[i]) { + prunned_branch_level[i] = cell_type - 1; + } + bit_reader.advance(4); + + cell_refs_cnt[i] = bit_reader.bits().get_uint(4); + bit_reader.advance(4); + if (cell_refs_cnt[i] > 4) { + return td::Status::Error("BOC decompression failed: invalid cell refs count"); + } + + if (prunned_branch_level[i]) { + size_t coef = std::bitset<4>(prunned_branch_level[i]).count(); + cell_data_length[i] = (256 + 16) * coef; + } else { + // Check enough bits for data length metadata + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for data length"); + } + + is_data_small[i] = bit_reader.bits().get_uint(1); + bit_reader.advance(1); + cell_data_length[i] = bit_reader.bits().get_uint(7); + bit_reader.advance(7); + + if (!is_data_small[i]) { + cell_data_length[i] *= 8; + if (!cell_data_length[i]) { + cell_data_length[i] += 1024; + } + } + } + + // Validate cell data length + if (cell_data_length[i] > kMaxCellDataLengthBits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + } + + // Read direct edge connections + for (int i = 0; i < node_count; ++i) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + boc_graph[i][j] = i + 1; + } + } + } + + // Read initial cell data + for (int i = 0; i < node_count; ++i) { + if (prunned_branch_level[i]) { + cell_builders[i].store_long((1 << 8) + prunned_branch_level[i], 16); + } + + size_t remainder_bits = cell_data_length[i] % 8; + if (bit_reader.size() < remainder_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for initial cell data"); + } + cell_builders[i].store_bits(bit_reader.subslice(0, remainder_bits)); + bit_reader.advance(remainder_bits); + cell_data_length[i] -= remainder_bits; + } + + // Decode remaining edge connections + for (size_t i = 0; i < node_count; ++i) { + if (node_count <= i + 3) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + boc_graph[i][j] = i + 2; + } + } + continue; + } + + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + size_t pref_size = (orig_size - bit_reader.size()); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); + + if (required_bits < 8 - (pref_size + 1) % 8 + 1) { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + pref_size = (orig_size - bit_reader.size()); + size_t available_bits = 8 - pref_size % 8; + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, available_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } + } + } + } + } + + // Check if all graph connections are valid + for (int node = 0; node < node_count; ++node) { + for (int j = 0; j < cell_refs_cnt[node]; ++j) { + size_t child_node = boc_graph[node][j]; + if (child_node >= node_count) { + return td::Status::Error("BOC decompression failed: invalid graph connection"); + } + if (child_node <= node) { + return td::Status::Error("BOC decompression failed: circular reference in graph"); + } + } + } + + // Align to byte boundary + while ((orig_size - bit_reader.size()) % 8) { + TRY_RESULT(bit, read_uint(bit_reader, 1)); + } + + // Read remaining cell data + for (int i = 0; i < node_count; ++i) { + size_t padding_bits = 0; + if (!prunned_branch_level[i] && !is_data_small[i]) { + while (bit_reader.size() > 0 && bit_reader.bits()[0] == 0) { + ++padding_bits; + bit_reader.advance(1); + } + TRY_RESULT(bit, read_uint(bit_reader, 1)); + ++padding_bits; + } + if (cell_data_length[i] < padding_bits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + size_t remaining_data_bits = cell_data_length[i] - padding_bits; + if (bit_reader.size() < remaining_data_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for remaining cell data"); + } + + cell_builders[i].store_bits(bit_reader.subslice(0, remaining_data_bits)); + bit_reader.advance(remaining_data_bits); + } + + // Build cell tree + std::vector> nodes(node_count); + for (int i = node_count - 1; i >= 0; --i) { + try { + for (int child_index = 0; child_index < cell_refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + cell_builders[i].store_ref(nodes[child]); + } + try { + nodes[i] = cell_builders[i].finalize(is_special[i]); + } catch (vm::CellBuilder::CellWriteError& e) { + return td::Status::Error("BOC decompression failed: write error while finalizing cell."); + } + } catch (vm::VmError& e) { + return td::Status::Error("BOC decompression failed: VM error during cell construction"); + } + } + + std::vector> root_nodes; + root_nodes.reserve(root_count); + for (size_t index : root_indexes) { + root_nodes.push_back(nodes[index]); + } + + return root_nodes; +} + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo) { + // Check for empty input + if (boc_roots.empty()) { + return td::Status::Error("Cannot compress empty BOC roots"); + } + + td::BufferSlice compressed; + if (algo == CompressionAlgorithm::BaselineLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_baseline_lz4(boc_roots)); + } else if (algo == CompressionAlgorithm::ImprovedStructureLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_improved_structure_lz4(boc_roots)); + } else { + return td::Status::Error("Unknown compression algorithm"); + } + + td::BufferSlice compressed_with_algo(compressed.size() + 1); + compressed_with_algo.data()[0] = int(algo); + memcpy(compressed_with_algo.data() + 1, compressed.data(), compressed.size()); + return compressed_with_algo; +} + +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size) { + if (compressed.size() == 0) { + return td::Status::Error("Can't decompress empty data"); + } + + int algo = int(compressed[0]); + compressed.remove_prefix(1); + + switch (algo) { + case int(CompressionAlgorithm::BaselineLZ4): + return boc_decompress_baseline_lz4(compressed, max_decompressed_size); + case int(CompressionAlgorithm::ImprovedStructureLZ4): + return boc_decompress_improved_structure_lz4(compressed, max_decompressed_size); + } + return td::Status::Error("Unknown compression algorithm"); +} + +} // namespace vm diff --git a/crypto/vm/boc-compression.h b/crypto/vm/boc-compression.h new file mode 100644 index 000000000..f143edd1f --- /dev/null +++ b/crypto/vm/boc-compression.h @@ -0,0 +1,42 @@ +/* +This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/CellBuilder.h" +#include "vm/excno.hpp" + +namespace vm { + +constexpr size_t kDecompressedSizeBytes = 4; + +enum class CompressionAlgorithm : int { BaselineLZ4 = 0, ImprovedStructureLZ4 = 1 }; + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size); + +} // namespace vm diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 72afb9988..93fc0a89d 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -89,14 +89,9 @@ td::Result CellSerializationInfo::get_bits(td::Slice cell) const { // TODO: check usage when result is empty td::Result> CellSerializationInfo::create_data_cell(td::Slice cell_slice, td::Span> refs) const { - CellBuilder cb; - TRY_RESULT(bits, get_bits(cell_slice)); - cb.store_bits(cell_slice.ubegin() + data_offset, bits); DCHECK(refs_cnt == (td::int64)refs.size()); - for (int k = 0; k < refs_cnt; k++) { - cb.store_ref(std::move(refs[k])); - } - TRY_RESULT(res, cb.finalize_novm_nothrow(special)); + TRY_RESULT(bits, get_bits(cell_slice)); + TRY_RESULT(res, DataCell::create(cell_slice.substr(data_offset), bits, refs, special)); CHECK(!res.is_null()); if (res->is_special() != special) { return td::Status::Error("is_special mismatch"); @@ -214,7 +209,7 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell is null"); } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } auto it = cells.find(cell->get_hash()); if (it != cells.end()) { @@ -560,7 +555,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } if (logger_ptr_) { @@ -593,7 +588,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } // std::cerr << std::endl; if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } writer.chk(); @@ -1161,6 +1156,16 @@ td::Result CellStorageStat::add_used_storage(Ref CellStorageStat::add_used_storage(td::Span> cells, bool kill_dup, + unsigned skip_count_root) { + CellInfo result; + for (const auto& cell : cells) { + TRY_RESULT(info, add_used_storage(cell, kill_dup, skip_count_root)); + result.max_merkle_depth = std::max(result.max_merkle_depth, info.max_merkle_depth); + } + return result; +} + void NewCellStorageStat::add_cell(Ref cell) { dfs(std::move(cell), true, false); } @@ -1259,35 +1264,55 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -static td::uint64 estimate_prunned_size() { - return 41; -} - -static td::uint64 estimate_serialized_size(const Ref& cell) { - return cell->get_serialized_size() + cell->size_refs() * 3 + 3; -} - -void ProofStorageStat::add_cell(const Ref& cell) { - auto& status = cells_[cell->get_hash()]; +void ProofStorageStat::add_loaded_cell(const Ref& cell, td::uint8 max_level) { + max_level = std::min(max_level, Cell::max_level); + auto& [status, size] = cells_[cell->get_hash(max_level)]; if (status == c_loaded) { return; } - if (status == c_prunned) { - proof_size_ -= estimate_prunned_size(); - } + proof_size_ -= size; status = c_loaded; - proof_size_ += estimate_serialized_size(cell); + proof_size_ += size = estimate_serialized_size(cell); + max_level += (cell->special_type() == CellTraits::SpecialType::MerkleProof || + cell->special_type() == CellTraits::SpecialType::MerkleUpdate); for (unsigned i = 0; i < cell->size_refs(); ++i) { - auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash(max_level)]; if (child_status == c_none) { child_status = c_prunned; - proof_size_ += estimate_prunned_size(); + proof_size_ += child_size = estimate_prunned_size(); } } } +void ProofStorageStat::add_loaded_cells(const ProofStorageStat& other) { + for (const auto& [hash, x] : other.cells_) { + const auto& [new_status, new_size] = x; + auto& [old_status, old_size] = cells_[hash]; + if (old_status >= new_status) { + continue; + } + proof_size_ -= old_size; + old_status = new_status; + proof_size_ += old_size = new_size; + } +} + + td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } +ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const { + auto it = cells_.find(hash); + return it == cells_.end() ? c_none : it->second.first; +} + +td::uint64 ProofStorageStat::estimate_prunned_size() { + return 41; +} + +td::uint64 ProofStorageStat::estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 17e7eb69d..5bf45c922 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -113,21 +113,19 @@ class NewCellStorageStat { struct CellStorageStat { unsigned long long cells; unsigned long long bits; - unsigned long long public_cells; struct CellInfo { td::uint32 max_merkle_depth = 0; }; td::HashMap seen; - CellStorageStat() : cells(0), bits(0), public_cells(0) { + CellStorageStat() : cells(0), bits(0) { } - explicit CellStorageStat(unsigned long long limit_cells) - : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) { } void clear_seen() { seen.clear(); } void clear() { - cells = bits = public_cells = 0; + cells = bits = 0; clear_limit(); clear_seen(); } @@ -145,6 +143,8 @@ struct CellStorageStat { td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(td::Span> cells, bool kill_dup = true, + unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); @@ -167,13 +167,21 @@ struct VmStorageStat { class ProofStorageStat { public: - void add_cell(const Ref& cell); + void add_loaded_cell(const Ref& cell, td::uint8 max_level = Cell::max_level); + void add_loaded_cells(const ProofStorageStat& other); td::uint64 estimate_proof_size() const; + + enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; + CellStatus get_cell_status(const Cell::Hash& hash) const; + bool is_loaded(const Cell::Hash& hash) const { + return get_cell_status(hash) == c_loaded; + } + + static td::uint64 estimate_prunned_size(); + static td::uint64 estimate_serialized_size(const Ref& cell); + private: - enum CellStatus { - c_none = 0, c_prunned = 1, c_loaded = 2 - }; - td::HashMap cells_; + td::HashMap> cells_; td::uint64 proof_size_ = 0; }; @@ -210,6 +218,7 @@ class BagOfCellsLogger { void start_stage(std::string stage) { log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + last_speed_log_ = td::Timestamp::now(); processed_cells_ = 0; timer_ = {}; stage_ = std::move(stage); @@ -217,15 +226,19 @@ class BagOfCellsLogger { void finish_stage(td::Slice desc) { LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc; } - td::Status on_cell_processed() { - ++processed_cells_; - if (processed_cells_ % 1000 == 0) { + td::Status on_cells_processed(size_t count) { + processed_cells_ += count; + if (processed_cells_ / 1000 > last_token_check_) { TRY_STATUS(cancellation_token_.check()); + last_token_check_ = processed_cells_ / 1000; } if (log_speed_at_.is_in_past()) { - log_speed_at_ += LOG_SPEED_PERIOD; - LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + double period = td::Timestamp::now().at() - last_speed_log_.at(); + + LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / period << " cells/s"; processed_cells_ = 0; + last_speed_log_ = td::Timestamp::now(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); } return td::Status::OK(); } @@ -236,6 +249,8 @@ class BagOfCellsLogger { td::CancellationToken cancellation_token_; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; + size_t last_token_check_ = 0; + td::Timestamp last_speed_log_; static constexpr double LOG_SPEED_PERIOD = 120.0; }; class BagOfCells { @@ -390,7 +405,7 @@ td::Result std_boc_serialize_multi(std::vector> root, td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 61ffe5c55..b76439fac 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -764,6 +764,15 @@ int exec_store_same(VmState* st, const char* name, int val) { return 0; } +int exec_builder_to_slice(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute BTOS"; + stack.check_underflow(1); + Ref cell = stack.pop_builder().write().finalize_novm(); + stack.push_cellslice(Ref{true, NoVm(), cell}); + return 0; +} + int exec_store_const_slice(VmState* st, CellSlice& cs, unsigned args, int pfx_bits) { unsigned refs = (args >> 3) & 3; unsigned data_bits = (args & 7) * 8 + 2; @@ -866,6 +875,7 @@ void register_cell_serialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xcf40, 16, "STZEROES", std::bind(exec_store_same, _1, "STZEROES", 0))) .insert(OpcodeInstr::mksimple(0xcf41, 16, "STONES", std::bind(exec_store_same, _1, "STONES", 1))) .insert(OpcodeInstr::mksimple(0xcf42, 16, "STSAME", std::bind(exec_store_same, _1, "STSAME", -1))) + .insert(OpcodeInstr::mksimple(0xcf50, 16, "BTOS", exec_builder_to_slice)->require_version(12)) .insert(OpcodeInstr::mkext(0xcf80 >> 7, 9, 5, dump_store_const_slice, exec_store_const_slice, compute_len_store_const_slice)); } diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index a75371dbb..25b71fcb2 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -35,15 +35,17 @@ namespace vm { using td::Ref; class DataCell; +struct LoadedCell { + Ref data_cell; + detail::VirtualizationParameters virt; + CellUsageTree::NodePtr tree_node; // TODO: inline_vector? +}; + class Cell : public CellTraits { public: using LevelMask = detail::LevelMask; using VirtualizationParameters = detail::VirtualizationParameters; - struct LoadedCell { - Ref data_cell; - VirtualizationParameters virt; - CellUsageTree::NodePtr tree_node; // TODO: inline_vector? - }; + using LoadedCell = vm::LoadedCell; using Hash = CellHash; static_assert(std::is_standard_layout::value, "Cell::Hash is not a standard layout type"); @@ -55,6 +57,7 @@ class Cell : public CellTraits { } // load interface + virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(VirtualizationParameters virt) const; virtual td::uint32 get_virtualization() const = 0; diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 772b5f6b7..a9ad449e1 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -50,7 +50,7 @@ Ref CellBuilder::finalize_copy(bool special) const { if (vm_state_interface) { vm_state_interface->register_cell_create(); } - auto res = DataCell::create(data, size(), td::span(refs.data(), size_refs()), special); + auto res = DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::span(refs.data(), size_refs()), special); if (res.is_error()) { LOG(DEBUG) << res.error(); throw CellWriteError{}; @@ -68,7 +68,8 @@ Ref CellBuilder::finalize_copy(bool special) const { } td::Result> CellBuilder::finalize_novm_nothrow(bool special) { - auto res = DataCell::create(data, size(), td::mutable_span(refs.data(), size_refs()), special); + auto res = + DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::mutable_span(refs.data(), size_refs()), special); bits = refs_cnt = 0; return res; } diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 954a1ac08..4c701c7d2 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -85,6 +85,9 @@ class CellBuilder : public td::CntObject { Ref get_ref(unsigned idx) const { return idx < refs_cnt ? refs[idx] : Ref{}; } + td::Span> get_refs() const { + return {refs.data(), refs_cnt}; + } void reset(); bool reset_bool() { reset(); diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 9cd3e931a..bea20f95d 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref& res) const { } } +std::vector> CellSlice::prefetch_all_refs() const { + std::vector> res(size_refs()); + for (unsigned i = 0; i < size_refs(); ++i) { + res[i] = prefetch_ref(i); + } + return res; +} + bool CellSlice::fetch_maybe_ref(Ref& res) { auto z = prefetch_ulong(1); if (!z) { diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index ecce30f5c..7525272b5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -190,6 +190,7 @@ class CellSlice : public td::CntObject { } bool fetch_maybe_ref(Ref& ref); bool prefetch_maybe_ref(Ref& ref) const; + std::vector> prefetch_all_refs() const; td::BitSlice fetch_bits(unsigned bits); td::BitSlice prefetch_bits(unsigned bits) const; td::Ref fetch_subslice(unsigned bits, unsigned refs = 0); diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 410b3fcd6..05a4f0c45 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -17,17 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/CellUsageTree.h" +#include "DataCell.h" namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { +bool CellUsageTree::NodePtr::on_load(const Cell::LoadedCell& loaded_cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_, cell); + tree->on_load(node_id_, loaded_cell); return true; } @@ -98,11 +99,11 @@ void CellUsageTree::mark_path(NodeId node_id) { } } -CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) { +CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) const { return nodes_[node_id].parent; } -CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) { +CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) const { DCHECK(ref_id < CellTraits::max_refs); return nodes_[node_id].children[ref_id]; } @@ -111,13 +112,13 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { - if (nodes_[node_id].is_loaded) { +void CellUsageTree::on_load(NodeId node_id, const Cell::LoadedCell& loaded_cell) { + if (ignore_loads_ || nodes_[node_id].is_loaded) { return; } nodes_[node_id].is_loaded = true; if (cell_load_callback_) { - cell_load_callback_(cell); + cell_load_callback_(loaded_cell); } } diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index af0f21f53..cd2470dda 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -27,6 +27,7 @@ namespace vm { class DataCell; +struct LoadedCell; class CellUsageTree : public std::enable_shared_from_this { public: @@ -42,7 +43,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load(const td::Ref& cell) const; + bool on_load(const LoadedCell& loaded_cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -58,14 +59,21 @@ class CellUsageTree : public std::enable_shared_from_this { bool has_mark(NodeId node_id) const; void set_mark(NodeId node_id, bool mark = true); void mark_path(NodeId node_id); - NodeId get_parent(NodeId node_id); - NodeId get_child(NodeId node_id, unsigned ref_id); + NodeId get_parent(NodeId node_id) const; + NodeId get_child(NodeId node_id, unsigned ref_id) const; void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { cell_load_callback_ = std::move(f); } + void set_ignore_loads(bool value) { + if (value) { + ++ignore_loads_; + } else { + --ignore_loads_; + } + } private: struct Node { @@ -76,9 +84,10 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; - std::function&)> cell_load_callback_; + std::function cell_load_callback_; - void on_load(NodeId node_id, const td::Ref& cell); + void on_load(NodeId node_id, const LoadedCell& loaded_cell); NodeId create_node(NodeId parent); + int ignore_loads_ = 0; }; } // namespace vm diff --git a/crypto/vm/cells/CellWithStorage.h b/crypto/vm/cells/CellWithStorage.h deleted file mode 100644 index f3fedfc31..000000000 --- a/crypto/vm/cells/CellWithStorage.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -namespace vm { -namespace detail { - -template -struct DefaultAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - return std::make_unique(std::forward(args)...); - } -}; - -template -class CellWithArrayStorage : public CellT { - public: - template - CellWithArrayStorage(ArgsT&&... args) : CellT(std::forward(args)...) { - } - ~CellWithArrayStorage() { - CellT::destroy_storage(get_storage()); - } - template - static auto create(Allocator allocator, size_t storage_size, ArgsT&&... args) { - static_assert(CellT::max_storage_size <= 40 * 8, ""); - //size = 128 + 32 + 8; - auto size = (storage_size + 7) / 8; -#define CASE(size) \ - case (size): \ - return allocator. template make_unique>(std::forward(args)...); -#define CASE2(offset) CASE(offset) CASE(offset + 1) -#define CASE8(offset) CASE2(offset) CASE2(offset + 2) CASE2(offset + 4) CASE2(offset + 6) -#define CASE32(offset) CASE8(offset) CASE8(offset + 8) CASE8(offset + 16) CASE8(offset + 24) - switch (size) { CASE32(0) CASE8(32) } -#undef CASE -#undef CASE2 -#undef CASE8 -#undef CASE32 - LOG(FATAL) << "TOO BIG " << storage_size; - UNREACHABLE(); - } - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return create(DefaultAllocator{}, storage_size, std::forward(args)...); - } - - private: - alignas(alignof(void*)) char storage_[Size]; - - const char* get_storage() const final { - return storage_; - } - char* get_storage() final { - return storage_; - } -}; - -template -class CellWithUniquePtrStorage : public CellT { - public: - template - CellWithUniquePtrStorage(size_t storage_size, ArgsT&&... args) - : CellT(std::forward(args)...), storage_(std::make_unique(storage_size)) { - } - ~CellWithUniquePtrStorage() { - CellT::destroy_storage(get_storage()); - } - - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return std::make_unique(storage_size, std::forward(args)...); - } - - private: - std::unique_ptr storage_; - - const char* get_storage() const final { - CHECK(storage_); - return storage_.get(); - } - char* get_storage() final { - CHECK(storage_); - return storage_.get(); - } -}; -} // namespace detail -} // namespace vm diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 4dd301616..b6ff8021f 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -16,336 +16,378 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/DataCell.h" #include "openssl/digest.hpp" - -#include "td/utils/ScopeGuard.h" - -#include "vm/cells/CellWithStorage.h" +#include "vm/cells/DataCell.h" namespace vm { -thread_local bool DataCell::use_arena = false; namespace { -template -struct ArenaAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - auto* ptr = fast_alloc(sizeof(T)); - T* obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } -private: - td::MutableSlice alloc_batch() { - size_t batch_size = 1 << 20; - auto batch = std::make_unique(batch_size); - return td::MutableSlice(batch.release(), batch_size); + +class CellChecker { + public: + CellChecker(bool is_special, td::Slice data, int bit_length, td::Span> refs) + : is_special_(is_special) + , refs_(refs) + , refs_cnt_(static_cast(refs.size())) + , data_(data) + , bit_length_(bit_length) { } - char* fast_alloc(size_t size) { - thread_local td::MutableSlice batch; - auto aligned_size = (size + 7) / 8 * 8; - if (batch.size() < size) { - batch = alloc_batch(); + + td::Status check_and_compute_level_info() { + // First, we figure out what is the type of the cell. + type_ = Cell::SpecialType::Ordinary; + + if (is_special_) { + if (bit_length_ < 8) { + return td::Status::Error("Not enough data for a special cell"); + } + + type_ = static_cast(read_byte(0)); + if (type_ == Cell::SpecialType::Ordinary) { + return td::Status::Error("Invalid special cell type"); + } } - auto res = batch.begin(); - batch.remove_prefix(aligned_size); - return res; - } -}; -} -std::unique_ptr DataCell::create_empty_data_cell(Info info) { - if (use_arena) { - ArenaAllocator allocator; - auto res = detail::CellWithArrayStorage::create(allocator, info.get_storage_size(), info); - // this is dangerous - Ref(res.get()).release(); - return res; + + // Next, we populate everything except for virtualization and hashes. `check_*` functions also + // perform type-specific checks. + switch (type_) { + case Cell::SpecialType::Ordinary: + TRY_STATUS(check_ordinary_cell()); + break; + case Cell::SpecialType::PrunnedBranch: + TRY_STATUS(check_pruned_branch()); + break; + case Cell::SpecialType::Library: + TRY_STATUS(check_library()); + break; + case Cell::SpecialType::MerkleProof: + TRY_STATUS(check_merkle_proof()); + break; + case Cell::SpecialType::MerkleUpdate: + TRY_STATUS(check_merkle_update()); + break; + default: + return td::Status::Error("Invalid special cell type"); + } + + // Afterwards, we do some common checks and compute virtualization level. + if (*std::max_element(depth_.begin(), depth_.end()) > CellTraits::max_depth) { + return td::Status::Error("Depth is too big"); + } + + for (int i = 0; i < refs_cnt_; ++i) { + virtualization_ = std::max(virtualization_, refs_[i]->get_virtualization()); + } + if (virtualization_ > std::numeric_limits::max()) { + return td::Status::Error("Virtualization is too big to be stored in vm::DataCell"); + } + + // And finally, we compute cell hashes. + // NOTE: Hash computation algorithm is not described correctly (or at all) in the documentation. + int last_computed_hash = -1; + + for (int i = 0; i <= max_level; ++i) { + if (!level_mask_.is_significant(i + 1) && i != max_level) { + continue; + } + + compute_hash(i, last_computed_hash); + for (int j = last_computed_hash + 1; j < i; ++j) { + hash_[j] = hash_[i]; + } + last_computed_hash = i; + } + + return {}; } - return detail::CellWithUniquePtrStorage::create(info.get_storage_size(), info); -} + // Getters for computed values + Cell::SpecialType type() const { + return type_; + } -DataCell::DataCell(Info info) : info_(std::move(info)) { - get_thread_safe_counter().add(1); -} -DataCell::~DataCell() { - get_thread_safe_counter().add(-1); -} + Cell::LevelMask level_mask() const { + return level_mask_; + } -void DataCell::destroy_storage(char* storage) { - auto* refs = info_.get_refs(storage); - for (size_t i = 0; i < get_refs_cnt(); i++) { - Ref(refs[i], Ref::acquire_t{}); // call destructor + td::uint8 virtualization() const { + return static_cast(virtualization_); } -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::Span> refs, - bool special) { - std::array, max_refs> copied_refs; - CHECK(refs.size() <= copied_refs.size()); - for (size_t i = 0; i < refs.size(); i++) { - copied_refs[i] = refs[i]; + std::array const& depths() const { + return depth_; } - return create(std::move(data), bits, td::MutableSpan>(copied_refs.data(), refs.size()), special); -} -DataCell::SpecialType DataCell::special_type() const { - if (is_special()) { - return static_cast(td::bitstring::bits_load_ulong(get_data(), 8)); + std::array const& hashes() const { + return hash_; } - return SpecialType::Ordinary; -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special) { - for (auto& ref : refs) { - if (ref.is_null()) { - return td::Status::Error("Has null cell reference"); - } + private: + static constexpr int max_level = CellTraits::max_level; + + static constexpr int hash_bytes = CellTraits::hash_bytes; + static_assert(hash_bytes == sizeof(CellHash)); + + static constexpr int depth_bytes = CellTraits::depth_bytes; + static_assert(depth_bytes == 2); + + td::uint8 read_byte(size_t i) { + return data_[i]; } - SpecialType type = SpecialType::Ordinary; - if (special) { - if (bits < 8) { - return td::Status::Error("Not enough data for a special cell"); + td::Status check_ordinary_cell() { + for (int i = 0; i < refs_cnt_; ++i) { + level_mask_ = level_mask_.apply_or(refs_[i]->get_level_mask()); + + for (int j = 0; j <= max_level; ++j) { + depth_[j] = std::max(depth_[j], refs_[i]->get_depth(j)); + } } - type = static_cast(td::bitstring::bits_load_ulong(data, 8)); - if (type == SpecialType::Ordinary) { - return td::Status::Error("Special cell has Ordinary type"); + + if (refs_cnt_ != 0) { + for (auto& depth : depth_) { + ++depth; + } } + + return {}; } - LevelMask level_mask; - td::uint32 virtualization = 0; - switch (type) { - case SpecialType::Ordinary: { - for (auto& ref : refs) { - level_mask = level_mask.apply_or(ref->get_level_mask()); - virtualization = td::max(virtualization, ref->get_virtualization()); - } - break; + td::Status check_pruned_branch() { + if (refs_cnt_ != 0) { + return td::Status::Error("Pruned branch cannot have references"); + } + if (bit_length_ < 16) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::PrunnedBranch: { - if (refs.size() != 0) { - return td::Status::Error("PrunnedBranch special cell has a cell reference"); - } - if (bits < 16) { - return td::Status::Error("Not enough data for a PrunnedBranch special cell"); - } - level_mask = LevelMask((td::bitstring::bits_load_ulong(data + 8, 8)) & 0xff); - auto level = level_mask.get_level(); - if (level > max_level || level == 0) { - return td::Status::Error("Prunned Branch has an invalid level"); - } - if (bits != (2 + level_mask.apply(level - 1).get_hashes_count() * (hash_bytes + depth_bytes)) * 8) { - return td::Status::Error("Not enouch data for a PrunnedBranch special cell"); - } - // depth will be checked later! - break; + level_mask_ = Cell::LevelMask{read_byte(1)}; + if (level_mask_.get_level() == 0 || level_mask_.get_level() > max_level) { + return td::Status::Error("Invalid level mask in a pruned branch"); } - case SpecialType::Library: { - if (bits != 8 + hash_bytes * 8) { - return td::Status::Error("Not enouch data for a Library special cell"); - } - if (!refs.empty()) { - return td::Status::Error("Library special cell has a cell reference"); - } - break; + int hashes_count = level_mask_.get_hash_i(); + auto expected_byte_size = 2 + hashes_count * (hash_bytes + depth_bytes); + + if (bit_length_ != static_cast(expected_byte_size * 8)) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::MerkleProof: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8) { - return td::Status::Error("Not enouch data for a MerkleProof special cell"); - } - if (refs.size() != 1) { - return td::Status::Error("Wrong references count for a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("Hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("Depth mismatch in a MerkleProof special cell"); + // depth[max_level] = 0; + + for (int i = max_level; i--;) { + if (level_mask_.is_significant(i + 1)) { + int hashes_before = level_mask_.apply(i).get_hash_i(); + auto offset = 2 + hashes_count * hash_bytes + hashes_before * depth_bytes; + depth_[i] = DataCell::load_depth(data_.ubegin() + offset); + } else { + depth_[i] = depth_[i + 1]; } - level_mask = refs[0]->get_level_mask().shift_right(); - virtualization = refs[0]->get_virtualization(); - break; } - case SpecialType::MerkleUpdate: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8 * 2) { - return td::Status::Error("Not enouch data for a MerkleUpdate special cell"); - } - if (refs.size() != 2) { - return td::Status::Error("Wrong references count for a MerkleUpdate special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("First hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8 + hash_bits, refs[1]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != - 0) { - return td::Status::Error("Second hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("First depth mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits + depth_bytes * 8, depth_bytes * 8) != - refs[1]->get_depth(0)) { - return td::Status::Error("Second depth mismatch in a MerkleProof special cell"); - } + return {}; + } - level_mask = refs[0]->get_level_mask().apply_or(refs[1]->get_level_mask()).shift_right(); - virtualization = td::max(refs[0]->get_virtualization(), refs[1]->get_virtualization()); - break; + td::Status check_library() { + if (refs_cnt_ != 0) { + return td::Status::Error("Library cell cannot have references"); + } + if (bit_length_ != 8 * (1 + hash_bytes)) { + return td::Status::Error("Length mismatch in a library cell"); } - default: - return td::Status::Error("Unknown special cell type"); + return {}; } - Info info; - if (td::unlikely(bits > max_bits)) { - return td::Status::Error("Too many bits"); - } - if (td::unlikely(refs.size() > max_refs)) { - return td::Status::Error("Too many cell references"); - } - if (td::unlikely(virtualization > max_virtualization)) { - return td::Status::Error("Too big virtualization"); - } + td::Status check_merkle_child(int child_idx, int hash_offset, int depth_offset) { + CellHash stored_hash; + std::memcpy(&stored_hash, data_.begin() + hash_offset, hash_bytes); + if (stored_hash != refs_[child_idx]->get_hash(0)) { + return td::Status::Error("Invalid hash in a Merkle proof or update"); + } - CHECK(level_mask.get_level() <= max_level); - - auto hash_count = type == SpecialType::PrunnedBranch ? 1 : level_mask.get_hashes_count(); - DCHECK(hash_count <= max_level + 1); - - info.bits_ = bits; - info.refs_count_ = refs.size() & 7; - info.is_special_ = special; - info.level_mask_ = level_mask.get_mask() & 7; - info.hash_count_ = hash_count & 7; - info.virtualization_ = virtualization & 7; - - auto data_cell = create_empty_data_cell(info); - auto* storage = data_cell->get_storage(); - - // init data - auto* data_ptr = info.get_data(storage); - td::BitPtr{data_ptr}.copy_from(data, bits); - // prepare for serialization - if (bits & 7) { - int m = (0x80 >> (bits & 7)); - unsigned l = bits / 8; - data_ptr[l] = static_cast((data_ptr[l] & -m) | m); - } + td::uint16 stored_depth = DataCell::load_depth(data_.ubegin() + depth_offset); + if (stored_depth != refs_[child_idx]->get_depth(0)) { + return td::Status::Error("Invalid depth in a Merkle proof or update"); + } - // init refs - auto refs_ptr = info.get_refs(storage); - for (size_t i = 0; i < refs.size(); i++) { - refs_ptr[i] = refs[i].release(); - } + for (int i = 0; i <= max_level; ++i) { + depth_[i] = std::max(depth_[i], refs_[child_idx]->get_depth(i + 1) + 1); + } - // init hashes and depth - auto* hashes_ptr = info.get_hashes(storage); - auto* depth_ptr = info.get_depth(storage); + return {}; + } - // NB: be careful with special cells - auto total_hash_count = level_mask.get_hashes_count(); - auto hash_i_offset = total_hash_count - hash_count; - for (td::uint32 level_i = 0, hash_i = 0, level = level_mask.get_level(); level_i <= level; level_i++) { - if (!level_mask.is_significant(level_i)) { - continue; + td::Status check_merkle_proof() { + if (refs_cnt_ != 1) { + return td::Status::Error("Merkle proof must have exactly one reference"); } - SCOPE_EXIT { - hash_i++; - }; - if (hash_i < hash_i_offset) { - continue; + if (bit_length_ != 8 * (1 + hash_bytes + depth_bytes)) { + return td::Status::Error("Length mismatch in a Merkle proof"); } - unsigned char tmp[2]; - tmp[0] = info.d1(level_mask.apply(level_i)); - tmp[1] = info.d2(); - static TD_THREAD_LOCAL digest::SHA256* hasher; - td::init_thread_local(hasher); - hasher->reset(); + TRY_STATUS(check_merkle_child(0, 1, 1 + hash_bytes)); - hasher->feed(td::Slice(tmp, 2)); + level_mask_ = refs_[0]->get_level_mask().shift_right(); - if (hash_i == hash_i_offset) { - DCHECK(level_i == 0 || type == SpecialType::PrunnedBranch); - hasher->feed(td::Slice(data_ptr, (bits + 7) >> 3)); - } else { - DCHECK(level_i != 0 && type != SpecialType::PrunnedBranch); - hasher->feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); + return {}; + } + + td::Status check_merkle_update() { + if (refs_cnt_ != 2) { + return td::Status::Error("Merkle update must have exactly two references"); + } + if (bit_length_ != 8 * (1 + (hash_bytes + depth_bytes) * 2)) { + return td::Status::Error("Length mismatch in a Merkle update"); } - auto dest_i = hash_i - hash_i_offset; + TRY_STATUS(check_merkle_child(0, 1, 1 + 2 * hash_bytes)); + TRY_STATUS(check_merkle_child(1, 1 + hash_bytes, 1 + 2 * hash_bytes + 2)); - // calc depth - td::uint16 depth = 0; - for (int i = 0; i < info.refs_count_; i++) { - td::uint16 child_depth = 0; - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - child_depth = refs_ptr[i]->get_depth(level_i + 1); - } else { - child_depth = refs_ptr[i]->get_depth(level_i); - } + level_mask_ = refs_[0]->get_level_mask().apply_or(refs_[1]->get_level_mask()).shift_right(); - // add depth into hash - td::uint8 child_depth_buf[depth_bytes]; - store_depth(child_depth_buf, child_depth); - hasher->feed(td::Slice(child_depth_buf, depth_bytes)); + return {}; + } - depth = std::max(depth, child_depth); + void compute_hash(int level, int last_computed_hash) { + if (level != max_level && type_ == Cell::SpecialType::PrunnedBranch) { + int hashes_before = level_mask_.apply(level).get_hash_i(); + auto offset = 2 + hashes_before * hash_bytes; + std::memcpy(&hash_[level], data_.begin() + offset, hash_bytes); + return; } - if (info.refs_count_ != 0) { - if (depth >= max_depth) { - return td::Status::Error("Depth is too big"); + + static_assert(2 + CellTraits::max_bytes + CellTraits::max_refs * (hash_bytes + depth_bytes) <= 512); + char data_to_hash[512]; + int pointer = 0; + + auto add_byte_to_hash = [&](char byte) { data_to_hash[pointer++] = byte; }; + + auto add_slice_to_hash = [&](td::Slice slice) { + std::memcpy(data_to_hash + pointer, slice.data(), slice.size()); + pointer += slice.size(); + }; + + auto d1 = refs_cnt_ + (is_special_ << 3) + (level_mask_.apply(level).get_mask() << 5); + add_byte_to_hash(static_cast(d1)); + auto d2 = (bit_length_ >> 3 << 1) + ((bit_length_ & 7) != 0); + add_byte_to_hash(static_cast(d2)); + + if (last_computed_hash != -1 && type_ != Cell::SpecialType::PrunnedBranch) { + add_slice_to_hash(hash_[last_computed_hash].as_slice()); + } else { + add_slice_to_hash(data_.substr(0, bit_length_ / 8)); + // If we are not byte-aligned, some bit gymnastics is required to correctly pad the last byte. + if (bit_length_ % 8 != 0) { + td::uint8 last_byte = data_[bit_length_ / 8]; + last_byte >>= 7 - bit_length_ % 8; + last_byte |= 1; + last_byte <<= 7 - bit_length_ % 8; + add_byte_to_hash(last_byte); } - depth++; } - depth_ptr[dest_i] = depth; - // children hash - for (int i = 0; i < info.refs_count_; i++) { - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - hasher->feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); - } else { - hasher->feed(refs_ptr[i]->get_hash(level_i).as_slice()); - } + bool is_merkle_node = type_ == Cell::SpecialType::MerkleUpdate || type_ == Cell::SpecialType::MerkleProof; + auto child_level = (is_merkle_node ? std::min(max_level, level + 1) : level); + + for (int i = 0; i < refs_cnt_; ++i) { + auto depth = refs_[i]->get_depth(child_level); + add_byte_to_hash((depth >> 8) & 255); + add_byte_to_hash((depth >> 0) & 255); + } + + for (int i = 0; i < refs_cnt_; ++i) { + add_slice_to_hash(refs_[i]->get_hash(child_level).as_slice()); } - auto extracted_size = hasher->extract(hashes_ptr[dest_i].as_slice()); - DCHECK(extracted_size == hash_bytes); + + digest::SHA256 hasher; + hasher.feed(data_to_hash, pointer); + hasher.extract(hash_[level].as_slice()); } - return Ref(data_cell.release(), Ref::acquire_t{}); + bool is_special_; + Cell::SpecialType type_; + td::Span> refs_; + int refs_cnt_; + td::Slice data_; + int bit_length_; + + Cell::LevelMask level_mask_; + td::uint32 virtualization_{0}; + std::array depth_{}; + std::array hash_{}; +}; + +char* allocate_in_arena(size_t size) { + constexpr size_t batch_size = 1 << 20; + thread_local td::MutableSlice batch; + + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size) { + batch = td::MutableSlice(new char[batch_size], batch_size); + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; } -const DataCell::Hash DataCell::do_get_hash(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return reinterpret_cast(info_.get_data(get_storage()) + 2)[hash_i]; - } - hash_i = 0; +} // namespace + +thread_local bool DataCell::use_arena = false; + +td::Result> DataCell::create(td::Slice data, int bit_length, td::Span> refs, bool is_special) { + CHECK(bit_length >= 0 && data.size() * 8 >= static_cast(bit_length)); + if (refs.size() > CellTraits::max_refs) { + return td::Status::Error("Too many references"); + } + if (bit_length > CellTraits::max_bits) { + return td::Status::Error("Too many data bits"); + } + + CellChecker checker{is_special, data, bit_length, refs}; + TRY_STATUS(checker.check_and_compute_level_info()); + + auto level_info_size = sizeof(detail::LevelInfo) * (checker.level_mask().get_level() + 1); + auto cell_size = sizeof(DataCell) + level_info_size + (bit_length + 7) / 8; + + void* storage = use_arena ? allocate_in_arena(cell_size) : ::operator new(cell_size); + DataCell* allocated_cell = new (storage) + DataCell{bit_length, refs.size(), checker.type(), checker.level_mask(), use_arena, checker.virtualization()}; + auto& cell = *allocated_cell; + + auto mutable_data = cell.trailer_ + level_info_size; + + std::memcpy(mutable_data, data.data(), (bit_length + 7) / 8); + if (bit_length % 8 != 0) { + auto& last_byte = mutable_data[bit_length / 8]; + // This is the same padding as was used in CellChecker::compute_hash above. + last_byte >>= (7 - bit_length % 8); + last_byte |= 1; + last_byte <<= (7 - bit_length % 8); + } + + auto level_info = new (cell.trailer_) detail::LevelInfo[checker.level_mask().get_level() + 1]; + + for (int i = 0; i <= cell.level_; ++i) { + level_info[i] = { + .hash = checker.hashes()[i], + .depth = checker.depths()[i], + }; + } + + for (int i = 0; i < cell.refs_cnt_; ++i) { + cell.refs_[i] = refs[i]; } - return info_.get_hashes(get_storage())[hash_i]; + + return Ref{allocated_cell, Ref::acquire_t{}}; } -td::uint16 DataCell::do_get_depth(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return load_depth(info_.get_data(get_storage()) + 2 + hash_bytes * this_hash_i + hash_i * depth_bytes); - } - hash_i = 0; +DataCell::~DataCell() { + for (size_t i = 0; i < level_ + 1; ++i) { + level_info()[i].~LevelInfo(); } - return info_.get_depth(get_storage())[hash_i]; + get_thread_safe_counter().add(-1); } int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) const { @@ -353,8 +395,8 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co if (len > buff_size) { return 0; } - buff[0] = static_cast(info_.d1() | (with_hashes * 16)); - buff[1] = info_.d2(); + buff[0] = static_cast(construct_d1(max_level) | (with_hashes * 16)); + buff[1] = construct_d2(); int hs = 0; if (with_hashes) { hs = (get_level_mask().get_hashes_count()) * (hash_bytes + depth_bytes); @@ -362,7 +404,7 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co std::memset(buff + 2, 0, hs); auto dest = td::MutableSlice(buff + 2, hs); auto level = get_level(); - // TODO: optimize for prunned brandh + // TODO: optimize for pruned branch for (unsigned i = 0; i <= level; i++) { if (!get_level_mask().is_significant(i)) { continue; @@ -377,7 +419,6 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co store_depth(dest.ubegin(), get_depth(i)); dest.remove_prefix(depth_bytes); } - // buff[2] = 0; // for testing hash verification in deserialization buff += hs; len -= hs; } @@ -401,8 +442,16 @@ std::string DataCell::to_hex() const { return hex_buff; } -std::ostream& operator<<(std::ostream& os, const DataCell& c) { - return os << c.to_hex(); +DataCell::DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, + bool allocated_in_arena, td::uint8 virtualization) + : bit_length_(bit_length) + , refs_cnt_(static_cast(refs_cnt)) + , type_(static_cast(type)) + , level_(static_cast(level_mask.get_level())) + , level_mask_(level_mask.get_mask()) + , allocated_in_arena_(allocated_in_arena) + , virtualization_(virtualization) { + get_thread_safe_counter().add(1); } } // namespace vm diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 6d3c845fc..172b3390a 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -17,215 +17,199 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" #include "td/utils/Span.h" - #include "td/utils/ThreadSafeCounter.h" +#include "vm/cells/Cell.h" namespace vm { -class DataCell : public Cell { +namespace detail { + +struct LevelInfo { + CellHash hash; + td::uint16 depth; +}; + +} // namespace detail + +class DataCell final : public Cell { public: // NB: cells created with use_arena=true are never freed static thread_local bool use_arena; - DataCell(const DataCell& other) = delete; - ~DataCell() override; + static td::Result> create(td::Slice data, int bit_length, td::Span> refs, bool is_special); static void store_depth(td::uint8* dest, td::uint16 depth) { td::bitstring::bits_store_long(dest, depth, depth_bits); } + static td::uint16 load_depth(const td::uint8* src) { return td::bitstring::bits_load_ulong(src, depth_bits) & 0xffff; } - protected: - struct Info { - unsigned bits_; - - // d1 - unsigned char refs_count_ : 3; - bool is_special_ : 1; - unsigned char level_mask_ : 3; - - unsigned char hash_count_ : 3; - - unsigned char virtualization_ : 3; - - unsigned char d1() const { - return d1(LevelMask{level_mask_}); - } - unsigned char d1(LevelMask level_mask) const { - // d1 = refs_count + 8 * is_special + 32 * level - // + 16 * with_hashes - for seriazlization - // d1 = 7 + 16 + 32 * l - for absent cells - return static_cast(refs_count_ + 8 * is_special_ + 32 * level_mask.get_mask()); - } - unsigned char d2() const { - auto res = static_cast((bits_ / 8) * 2); - if ((bits_ & 7) != 0) { - return static_cast(res + 1); - } - return res; - } - size_t get_hashes_offset() const { - return 0; - } - size_t get_refs_offset() const { - return get_hashes_offset() + hash_bytes * hash_count_; - } - size_t get_depth_offset() const { - return get_refs_offset() + refs_count_ * sizeof(Cell*); - } - size_t get_data_offset() const { - return get_depth_offset() + sizeof(td::uint16) * hash_count_; - } - size_t get_storage_size() const { - return get_data_offset() + (bits_ + 7) / 8; + void operator delete(DataCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->allocated_in_arena_; + ptr->~DataCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); } + } - const Hash* get_hashes(const char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } + DataCell(DataCell const&) = delete; + DataCell(DataCell&&) = delete; - Hash* get_hashes(char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } - - const td::uint16* get_depth(const char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); - } + ~DataCell(); - td::uint16* get_depth(char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); - } - - const unsigned char* get_data(const char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } - unsigned char* get_data(char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } + virtual td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } - Cell* const* get_refs(const char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - Cell** get_refs(char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - }; + virtual td::Result load_cell() const override { + return LoadedCell{ + .data_cell = Ref{this}, + .virt = {}, + .tree_node = {}, + }; + } - Info info_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - // TODO: we may also save three different pointers + virtual td::uint32 get_virtualization() const override { + return virtualization_; + } - void destroy_storage(char* storage); + virtual CellUsageTree::NodePtr get_tree_node() const override { + return {}; + } - explicit DataCell(Info info); + virtual bool is_loaded() const override { + return true; + } - public: - td::Result load_cell() const override { - return LoadedCell{Ref{this}, {}, {}}; + virtual LevelMask get_level_mask() const override { + return LevelMask{level_mask_}; } + unsigned get_refs_cnt() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned get_bits() const { - return info_.bits_; + return bit_length_; } + unsigned size_refs() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned size() const { - return info_.bits_; + return bit_length_; } - const unsigned char* get_data() const { - return info_.get_data(get_storage()); + + unsigned char const* get_data() const { + return reinterpret_cast(trailer_ + sizeof(detail::LevelInfo) * (level_ + 1)); } + Ref get_ref(unsigned idx) const { - if (idx >= get_refs_cnt()) { - return Ref{}; + if (idx >= refs_cnt_) { + return {}; } - return Ref(get_ref_raw_ptr(idx)); + return refs_[idx]; } Cell* get_ref_raw_ptr(unsigned idx) const { - DCHECK(idx < get_refs_cnt()); - return info_.get_refs(get_storage())[idx]; + DCHECK(idx < refs_cnt_); + return const_cast(refs_[idx].get()); } Ref reset_ref_unsafe(unsigned idx, Ref ref, bool check_hash = true) { CHECK(idx < get_refs_cnt()); - auto refs = info_.get_refs(get_storage()); - CHECK(!check_hash || refs[idx]->get_hash() == ref->get_hash()); - auto res = Ref(refs[idx], Ref::acquire_t{}); // call destructor - refs[idx] = ref.release(); - return res; + CHECK(!check_hash || refs_[idx]->get_hash() == ref->get_hash()); + return std::exchange(refs_[idx], std::move(ref)); } - td::uint32 get_virtualization() const override { - return info_.virtualization_; - } - CellUsageTree::NodePtr get_tree_node() const override { - return {}; - } - bool is_loaded() const override { - return true; - } - LevelMask get_level_mask() const override { - return LevelMask{info_.level_mask_}; + bool is_special() const { + return type_ != static_cast(SpecialType::Ordinary); } - bool is_special() const { - return info_.is_special_; + SpecialType special_type() const { + return static_cast(type_); } - SpecialType special_type() const; + int get_serialized_size(bool with_hashes = false) const { return ((get_bits() + 23) >> 3) + (with_hashes ? get_level_mask().get_hashes_count() * (hash_bytes + depth_bytes) : 0); } + size_t get_storage_size() const { - return info_.get_storage_size(); + return sizeof(DataCell) + sizeof(detail::LevelInfo) * (level_ + 1) + (bit_length_ + 7) / 8; } + int serialize(unsigned char* buff, int buff_size, bool with_hashes = false) const; + std::string serialize() const; + std::string to_hex() const; + static td::int64 get_total_data_cells() { return get_thread_safe_counter().sum(); } template void store(StorerT& storer) const { - storer.template store_binary(info_.d1()); - storer.template store_binary(info_.d2()); + storer.template store_binary(construct_d1(max_level)); + storer.template store_binary(construct_d2()); storer.store_slice(td::Slice(get_data(), (get_bits() + 7) / 8)); } - protected: - static constexpr auto max_storage_size = max_refs * sizeof(void*) + (max_level + 1) * hash_bytes + max_bytes; - private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DataCell"); return res; } - static std::unique_ptr create_empty_data_cell(Info info); - const Hash do_get_hash(td::uint32 level) const override; - td::uint16 do_get_depth(td::uint32 level) const override; + DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, bool allocated_in_arena, + td::uint8 virtualization); + + detail::LevelInfo const* level_info() const { + return reinterpret_cast(trailer_); + } + + virtual td::uint16 do_get_depth(td::uint32 level) const override { + return level_info()[std::min(level_, level)].depth; + } + + virtual const Hash do_get_hash(td::uint32 level) const override { + return level_info()[std::min(level_, level)].hash; + } + + td::uint8 construct_d1(td::uint32 level) const { + return static_cast(refs_cnt_ + (is_special() << 3) + (get_level_mask().apply(level).get_mask() << 5)); + } + + td::uint8 construct_d2() const { + return static_cast(bit_length_ / 8 + (bit_length_ + 7) / 8); + } + + unsigned bit_length_ : 11; + unsigned refs_cnt_ : 3; + unsigned type_ : 3; + unsigned level_ : 3; + unsigned level_mask_ : 3; + unsigned allocated_in_arena_ : 1; + unsigned virtualization_ : 8; - friend class CellBuilder; - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::Span> refs, bool special); - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special); + std::array, max_refs> refs_{}; + + alignas(detail::LevelInfo) char trailer_[]; }; -std::ostream& operator<<(std::ostream& os, const DataCell& c); +inline std::ostream& operator<<(std::ostream& os, const DataCell& c) { + return os << c.to_hex(); +} + inline CellHash as_cell_hash(const Ref& cell) { return cell->get_hash(); } } // namespace vm - diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index 401bb0483..dbbd8575b 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -65,6 +65,9 @@ class ExtCell : public Cell { bool is_loaded() const override { return CellView(this)->is_loaded(); } + Ref> get_prunned_cell() const { + return prunned_cell_.load(); + } private: mutable td::AtomicRef data_cell_; @@ -112,6 +115,23 @@ class ExtCell : public Cell { return CellView(this)->get_depth(level); } + td::Status set_data_cell(Ref&& new_data_cell) const override { + auto prunned_cell = prunned_cell_.load(); + if (prunned_cell.is_null()) { + auto old_data_cell = data_cell_.get_unsafe(); + DCHECK(old_data_cell); + TRY_STATUS(old_data_cell->check_equals_unloaded(new_data_cell)); + return td::Status::OK(); + } + + TRY_STATUS(prunned_cell->check_equals_unloaded(new_data_cell)); + if (data_cell_.store_if_empty(new_data_cell)) { + prunned_cell_.store({}); + get_thread_safe_counter_unloaded().add(-1); + } + return td::Status::OK(); + } + td::Result> load_data_cell() const { auto data_cell = data_cell_.get_unsafe(); if (data_cell) { diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index 26dff7879..4c1fcbdb7 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/MerkleProof.h" +#include "td/utils/Status.h" #include "vm/cells/CellBuilder.h" #include "vm/cells/CellSlice.h" #include "vm/boc.h" @@ -152,6 +153,11 @@ Ref MerkleProof::virtualize(Ref cell, int virtualization) { return virtualize_raw(r_raw.move_as_ok(), {0 /*level*/, static_cast(virtualization)}); } +td::Result> MerkleProof::try_virtualize(Ref cell, int virtualization) { + TRY_RESULT(unpacked_cell, unpack_proof(std::move(cell))); + return unpacked_cell->virtualize({0 /*level*/, static_cast(virtualization)}); +} + class MerkleProofCombineFast { public: MerkleProofCombineFast(Ref a, Ref b) : a_(std::move(a)), b_(std::move(b)) { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index fc2cb6ebd..68f0be7b9 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -36,6 +36,7 @@ class MerkleProof { // cell must have zero level and must be a MerkleProof static Ref virtualize(Ref cell, int virtualization); + static td::Result> try_virtualize(Ref cell, int virtualization = 1); static Ref combine(Ref a, Ref b); static td::Result> combine_status(Ref a, Ref b); @@ -63,13 +64,19 @@ class MerkleProofBuilder { Ref root() const { return usage_root; } + Ref original_root() const { + return orig_root; + } td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } + const CellUsageTree &get_usage_tree() const { + return *usage_tree; + } }; } // namespace vm diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index a58b245cc..f4d40c285 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/CellWithStorage.h" -#include "vm/cells/Cell.h" + +#include "vm/cells/DataCell.h" namespace vm { struct PrunnedCellInfo { @@ -28,7 +28,7 @@ struct PrunnedCellInfo { }; template -class PrunnedCell : public Cell { +class PrunnedCell final : public Cell { public: ExtraT& get_extra() { return extra_; @@ -37,22 +37,33 @@ class PrunnedCell : public Cell { return extra_; } + void operator delete(PrunnedCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->info_.allocated_in_arena_; + ptr->~PrunnedCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); + } + } + static td::Result>> create(const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { - return create(detail::DefaultAllocator>(), prunned_cell_info, std::forward(extra)); + auto allocator = [](size_t bytes) { return ::operator new(bytes); }; + return create(allocator, true, prunned_cell_info, std::move(extra)); } - template - static td::Result>> create(AllocatorT allocator, const PrunnedCellInfo& prunned_cell_info, - ExtraT&& extra) { + template + static td::Result>> create(AllocatorFunc&& allocator, bool should_free, + const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { auto level_mask = prunned_cell_info.level_mask; if (level_mask.get_level() > max_level) { return td::Status::Error("Level is too big"); } Info info(level_mask); - auto prunned_cell = - detail::CellWithArrayStorage>::create(allocator, info.get_storage_size(), info, std::move(extra)); - TRY_STATUS(prunned_cell->init(prunned_cell_info)); - return Ref>(prunned_cell.release(), typename Ref>::acquire_t{}); + + auto storage = allocator(sizeof(PrunnedCell) + info.get_storage_size()); + auto* result = new (storage) PrunnedCell{info, std::move(extra)}; + result->info_.allocated_in_arena_ = !should_free; + TRY_STATUS(result->init(prunned_cell_info)); + return Ref>(result, typename Ref>::acquire_t{}); } LevelMask get_level_mask() const override { @@ -68,6 +79,7 @@ class PrunnedCell : public Cell { } unsigned char level_mask_ : 3; unsigned char hash_count_ : 3; + unsigned char allocated_in_arena_ : 1; size_t get_hashes_offset() const { return 0; } @@ -93,16 +105,10 @@ class PrunnedCell : public Cell { Info info_; ExtraT extra_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - void destroy_storage(char* storage) { - // noop - } td::Status init(const PrunnedCellInfo& prunned_cell_info) { - auto storage = get_storage(); auto& new_hash = prunned_cell_info.hash; - auto* hash = info_.get_hashes(storage); + auto* hash = info_.get_hashes(trailer_); size_t n = prunned_cell_info.level_mask.get_hashes_count(); CHECK(new_hash.size() == n * hash_bytes); for (td::uint32 i = 0; i < n; i++) { @@ -111,7 +117,7 @@ class PrunnedCell : public Cell { auto& new_depth = prunned_cell_info.depth; CHECK(new_depth.size() == n * depth_bytes); - auto* depth = info_.get_depth(storage); + auto* depth = info_.get_depth(trailer_); for (td::uint32 i = 0; i < n; i++) { depth[i] = DataCell::load_depth(new_depth.substr(i * Cell::depth_bytes, Cell::depth_bytes).ubegin()); if (depth[i] > max_depth) { @@ -135,15 +141,21 @@ class PrunnedCell : public Cell { private: const Hash do_get_hash(td::uint32 level) const override { - return info_.get_hashes(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_hashes(trailer_)[get_level_mask().apply(level).get_hash_i()]; } td::uint16 do_get_depth(td::uint32 level) const override { - return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_depth(trailer_)[get_level_mask().apply(level).get_hash_i()]; + } + + td::Status set_data_cell(Ref &&data_cell) const override { + return td::Status::OK(); } td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } + + char trailer_[]; }; } // namespace vm diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 3e6e88982..c634bc909 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -36,10 +36,13 @@ class UsageCell : public Cell { return Ref{true, std::move(cell), std::move(tree_node), PrivateTag{}}; } + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load(loaded_cell.data_cell)) { + if (tree_node_.on_load(loaded_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index 02abc1c88..a75bdf9de 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -37,6 +37,9 @@ class VirtualCell : public Cell { } // load interface + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); loaded_cell.virt = loaded_cell.virt.apply(virt_); diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index 1ccf53daf..e4499aa02 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -214,7 +214,7 @@ int exec_ret_data(VmState* st) { // Mode: // +1 = same_c3 (set c3 to code) -// +2 = push_0 (push an implicit 0 before running the code) +// +2 = push_0 (push an implicit 0 before running the code); only works with +1 enabled // +4 = load c4 (persistent data) from stack and return its final value // +8 = load gas limit from stack and return consumed gas // +16 = load c7 (smart-contract context) diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 522c987be..a38980638 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -40,6 +40,17 @@ class CellHashTable { return res; } + template + std::pair emplace(td::Slice hash, ArgsT &&...args) { + auto it = set_.find(hash); + if (it != set_.end()) { + return std::pair(const_cast(*it), false); + } + auto res = set_.emplace(std::forward(args)...); + CHECK(res.second); + return std::pair(const_cast(*res.first), res.second); + } + template void for_each(F &&f) { for (auto &info : set_) { @@ -64,7 +75,7 @@ class CellHashTable { size_t size() const { return set_.size(); } - InfoT* get_if_exists(td::Slice hash) { + InfoT *get_if_exists(td::Slice hash) { auto it = set_.find(hash); if (it != set_.end()) { return &const_cast(*it); diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 06df461ef..33c18ab3f 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -17,18 +17,24 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/CellStorage.h" + +#include "td/utils/Parser.h" #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/boc.h" #include "td/utils/base64.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include + namespace vm { namespace { + class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) - : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level = vm::Cell::max_level) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc), max_level_(max_level) { + CHECK(!as_boc_ || max_level_ == vm::Cell::max_level); } template @@ -43,11 +49,28 @@ class RefcntCellStorer { storer.store_slice(data); return; } + CHECK(refcnt_ > 0); store(refcnt_, storer); - store(*cell_, storer); + CHECK(cell_.not_null()) + if (max_level_ == vm::Cell::max_level) { + store(*cell_, storer); + } else { + auto level_mask = cell_->get_level_mask().apply(max_level_); + storer.template store_binary( + static_cast(cell_->get_refs_cnt() + 8 * cell_->is_special() + 32 * level_mask.get_mask())); + auto d2 = static_cast((cell_->get_bits() / 8) * 2); + if ((cell_->get_bits() & 7) != 0) { + d2 = static_cast(d2 + 1); + } + storer.template store_binary(d2); + storer.store_slice(td::Slice(cell_->get_data(), (cell_->get_bits() + 7) / 8)); + } + for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); - auto level_mask = cell->get_level_mask(); + auto level_mask = + cell->get_level_mask().apply(max_level_ + (cell_->special_type() == CellTraits::SpecialType::MerkleProof || + cell_->special_type() == CellTraits::SpecialType::MerkleUpdate)); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); storer.store_slice(td::Slice(&x, 1)); @@ -72,6 +95,7 @@ class RefcntCellStorer { td::int32 refcnt_; td::Ref cell_; bool as_boc_; + int max_level_; }; class RefcntCellParser { @@ -91,6 +115,7 @@ class RefcntCellParser { stored_boc_ = true; parse(refcnt, parser); } + CHECK(refcnt > 0); if (!need_data_) { return; } @@ -112,13 +137,13 @@ class RefcntCellParser { Ref refs[Cell::max_refs]; for (int i = 0; i < info.refs_cnt; i++) { if (data.size() < 1) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } Cell::LevelMask level_mask(data[0]); auto n = level_mask.get_hashes_count(); auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes); if (data.size() < end_offset) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes), @@ -159,6 +184,9 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return LoadResult{}; } + if (serialized.empty()) { + return LoadResult{}; + } TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); if (on_load_callback_) { on_load_callback_(res); @@ -166,6 +194,28 @@ td::Result CellLoader::load(td::Slice hash, bool need_da return res; } +td::Result> CellLoader::load_bulk(td::Span hashes, bool need_data, + ExtCellCreator &ext_cell_creator) { + std::vector values; + TRY_RESULT(get_statuses, reader_->get_multi(hashes, &values)); + std::vector res; + res.reserve(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto get_status = get_statuses[i]; + if (get_status != KeyValue::GetStatus::Ok) { + DCHECK(get_status == KeyValue::GetStatus::NotFound); + res.push_back(LoadResult{}); + continue; + } + TRY_RESULT(load_res, load(hashes[i], values[i], need_data, ext_cell_creator)); + if (on_load_callback_) { + on_load_callback_(load_res); + } + res.push_back(std::move(load_res)); + } + return res; +} + td::Result CellLoader::load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator) { LoadResult res; @@ -198,6 +248,7 @@ td::Result CellLoader::load_refcnt(td::Slice hash) { if (res.refcnt_ == -1) { parse(res.refcnt_, parser); } + CHECK(res.refcnt_ > 0); TRY_STATUS(parser.get_status()); return res; } @@ -209,11 +260,84 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc, max_level)); } td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } + +td::Status CellStorer::merge(td::Slice hash, td::int32 refcnt_diff) { + return kv_.merge(hash, serialize_refcnt_diffs(refcnt_diff)); +} + +void CellStorer::merge_value_and_refcnt_diff(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + CHECK(left.size() > 4); + CHECK(right.size() == 4); + + td::int32 left_refcnt = td::as(left.data()); + size_t shift = 0; + if (left_refcnt == -1) { + CHECK(left.size() >= 8); + left_refcnt = td::as(left.data() + 4); + shift = 4; + } + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 new_refcnt = left_refcnt + right_refcnt_diff; + CHECK(new_refcnt > 0); + td::as(left.data() + shift) = new_refcnt; +} +void CellStorer::merge_refcnt_diffs(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + if (left.empty()) { + left = right.str(); + return; + } + CHECK(left.size() == 4); + CHECK(right.size() == 4); + td::int32 left_refcnt_diff = td::as(left.data()); + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 total_refcnt_diff = left_refcnt_diff + right_refcnt_diff; + td::as(left.data()) = total_refcnt_diff; +} + +std::string CellStorer::serialize_refcnt_diffs(td::int32 refcnt_diff) { + TD_PERF_COUNTER(cell_store_refcnt_diff); + std::string s(4, 0); + td::as(s.data()) = refcnt_diff; + return s; +} + +td::Status CellStorer::apply_diff(const Diff &diff) { + switch (diff.type) { + case Diff::Set: + return kv_.set(diff.key.as_slice(), diff.value); + case Diff::Erase: + return kv_.erase(diff.key.as_slice()); + case Diff::Merge: + return kv_.merge(diff.key.as_slice(), diff.value); + default: + UNREACHABLE(); + } +} +td::Status CellStorer::apply_meta_diff(const MetaDiff &diff) { + switch (diff.type) { + case MetaDiff::Set: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(!diff.value.empty()); + return kv_.set(diff.key, diff.value); + case MetaDiff::Erase: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(diff.value.empty()); + return kv_.erase(diff.key); + default: + UNREACHABLE(); + } +} } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index cabd7fdcb..e17fab5f2 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -49,8 +49,12 @@ class CellLoader { }; CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); + td::Result> load_bulk(td::Span hashes, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null + KeyValueReader &key_value_reader() const { + return *reader_; + } private: std::shared_ptr reader_; @@ -62,7 +66,28 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); - static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + td::Status merge(td::Slice hash, td::int32 refcnt_diff); + + static void merge_value_and_refcnt_diff(std::string &value, td::Slice right); + static void merge_refcnt_diffs(std::string &left, td::Slice right); + static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); + + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, + int max_level = vm::Cell::max_level); + + struct Diff { + enum Type { Set, Erase, Merge } type{Set}; + CellHash key; + std::string value{}; + }; + td::Status apply_diff(const Diff &diff); + + struct MetaDiff { + enum Type { Set, Erase } type{Set}; + std::string key; + std::string value{}; + }; + td::Status apply_meta_diff(const MetaDiff &diff); private: KeyValue &kv_; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 093037583..697e4d67f 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -66,19 +66,27 @@ struct CellInfo { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} - bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} - bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} - + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } }; }; - bool operator<(const CellInfo &a, td::Slice b) { return a.key().as_slice() < b; } @@ -99,6 +107,36 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } + td::Result>> meta_get_all(size_t max_count) const override { + std::vector> result; + auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", + [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + return loader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } td::Result> load_cell(td::Slice hash) override { auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { @@ -116,6 +154,18 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> load_root(td::Slice hash) override { return load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto cell = load_cell(hash); + if (cell.is_error()) { + return cell.move_as_error(); + } + result.push_back(cell.move_as_ok()); + } + return result; + } td::Result> load_root_thread_safe(td::Slice hash) const override { return td::Status::Error("Not implemented"); } @@ -198,21 +248,29 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } + td::PerfWarningTimer timer_dfs_new_cells_in_db("dfs_new_cells_in_db"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } + timer_dfs_new_cells_in_db.reset(); + td::PerfWarningTimer timer_dfs_new_cells("dfs_new_cells"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } + timer_dfs_new_cells.reset(); + td::PerfWarningTimer timer_dfs_old_cells("dfs_old_cells"); for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } + timer_dfs_old_cells.reset(); + td::PerfWarningTimer timer_save_diff_prepare("save_diff_prepare"); save_diff_prepare(); + timer_save_diff_prepare.reset(); to_inc_.clear(); to_dec_.clear(); @@ -222,6 +280,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Status commit(CellStorer &storer) override { prepare_commit(); + td::PerfWarningTimer times_save_diff("save diff", 0.01); save_diff(storer); // Some elements are erased from hash table, to keep it small. // Hash table is no longer represents the difference between the loader and @@ -249,7 +308,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -259,6 +318,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat std::vector> to_dec_; CellHashTable hash_table_; std::vector visited_; + std::vector meta_diffs_; Stats stats_diff_; td::uint32 celldb_compress_depth_{0}; @@ -269,8 +329,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -279,7 +340,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -334,6 +395,23 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(load_result.cell()); } + td::Result>> load_bulk(td::Span hashes) override { + if (db_) { + return db_->load_bulk(hashes); + } + TRY_RESULT(load_result, cell_loader_->load_bulk(hashes, true, *this)); + + std::vector> res; + res.reserve(load_result.size()); + for (auto &load_res : load_result) { + if (load_res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + res.push_back(std::move(load_res.cell())); + } + return res; + } + private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); @@ -382,8 +460,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); @@ -441,6 +518,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } + for (auto meta_diff : meta_diffs_) { + storer.apply_meta_diff(meta_diff); + } + meta_diffs_.clear(); visited_.clear(); } @@ -558,6 +639,8 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } auto res = r_res.move_as_ok(); if (res.status != CellLoader::LoadResult::Ok) { + LOG_CHECK(info.cell.not_null()) << "Trying to load nonexistent cell from db " + << CellHash::from_slice(hash).to_hex(); break; } info.cell = std::move(res.cell()); @@ -651,7 +734,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable cells_; - std::queue load_queue_; + std::queue load_queue_; td::uint32 active_load_ = 0; td::uint32 max_parallel_load_ = 4; }; @@ -814,11 +897,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat pca_state_->promise_.set_result(td::Unit()); pca_state_ = {}; } - }; } // namespace -std::unique_ptr DynamicBagOfCellsDb::create() { +std::unique_ptr DynamicBagOfCellsDb::create(CreateV1Options) { return std::make_unique(); } } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 62864ad97..ec0aee9c1 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/db/KeyValue.h" #include "vm/cells.h" #include "td/utils/Slice.h" @@ -44,18 +45,30 @@ class CellDbReader { public: virtual ~CellDbReader() = default; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; }; class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; + + virtual td::Result>> meta_get_all(size_t max_count) const = 0; + virtual td::Result meta_get(td::Slice key, std::string &value) = 0; + virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; + virtual td::Status meta_erase(td::Slice key) = 0; + virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; + virtual td::Result>> load_known_roots() const { + return std::vector>(); + } struct Stats { td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; + td::NamedStats named_stats; std::vector> custom_stats; void apply_diff(const Stats &diff) { roots_total_count += diff.roots_total_count; @@ -64,6 +77,20 @@ class DynamicBagOfCellsDb { CHECK(roots_total_count >= 0); CHECK(cells_total_count >= 0); CHECK(cells_total_size >= 0); + named_stats.apply_diff(diff.named_stats); + } + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const Stats &stats) { + sb << "STATS\n"; + for (auto &p : stats.custom_stats) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_int) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_str) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + return sb; } }; virtual void inc(const Ref &old_root) = 0; @@ -72,7 +99,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Result get_stats() { - return td::Status::Error("Not implemented"); + return Stats{}; } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -83,25 +110,49 @@ class DynamicBagOfCellsDb { virtual void set_celldb_compress_depth(td::uint32 value) = 0; virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; - static std::unique_ptr create(); + class AsyncExecutor { + public: + virtual ~AsyncExecutor() { + } + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + virtual std::string describe() const { + return "AsyncExecutor"; + } + }; + + struct CreateV1Options { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV1Options &options) { + return sb << "V1{}"; + } + }; + static std::unique_ptr create(CreateV1Options = {}); + + struct CreateV2Options { + size_t extra_threads{std::thread::hardware_concurrency()}; + std::shared_ptr executor{}; + size_t cache_ttl_max{2000}; + size_t cache_size_max{1000000}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV2Options &options) { + return sb << "V2{extra_threads=" << options.extra_threads << ", cache_ttl_max=" << options.cache_ttl_max + << ", cache_size_max=" << options.cache_size_max << "}"; + } + }; + static std::unique_ptr create_v2(CreateV2Options options); struct CreateInMemoryOptions { size_t extra_threads{std::thread::hardware_concurrency()}; bool verbose{true}; - // Allocated DataCels will never be deleted + // Allocated DataCells will never be deleted bool use_arena{false}; // Almost no overhead in memory during creation, but will scan database twice bool use_less_memory_during_creation{true}; - }; - static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); - - class AsyncExecutor { - public: - virtual ~AsyncExecutor() { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateInMemoryOptions &options) { + return sb << "InMemory{extra_threads=" << options.extra_threads << ", use_arena=" << options.use_arena + << ", use_less_memory_during_creation=" << options.use_less_memory_during_creation << "}"; } - virtual void execute_async(std::function f) = 0; - virtual void execute_sync(std::function f) = 0; }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp new file mode 100644 index 000000000..3b7d7c233 --- /dev/null +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -0,0 +1,1531 @@ +#include "vm/db/DynamicBagOfCellsDb.h" +#include "vm/db/CellStorage.h" +#include "vm/db/CellHashTable.h" + +#include "vm/cells/ExtCell.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/ThreadSafeCounter.h" +#include "td/utils/misc.h" +#include "validator/validator.h" + +#include "vm/cellslice.h" + +#include + +namespace vm { +namespace { + +// Very stupid Vector/MpmcQueue +template +struct TsVector { + TsVector() { + first_block_size_ = 64; + blocks_[0].data.resize(first_block_size_); + blocks_[0].is_ready = true; + } + TsVector(std::vector base) { + first_block_size_ = base.size(); + blocks_[0].data = std::move(base); + blocks_[0].is_ready = true; + } + struct Block { + std::mutex mutex; + std::atomic is_ready{false}; + std::vector data; + }; + T &at(size_t i) { + td::uint64 j = i / first_block_size_; + td::int32 hb = 63 - td::count_leading_zeroes64(j); // hb = -1 if j=0, else hb>=0 + + // If j=0, hb<0, so hb>>31 = -1 => mask=0 + // If j>0, hb>=0, so hb>>31=0 => mask=~0 (all ones) + td::uint64 mask = ~(td::uint64)(hb >> 31); + + size_t block_i = hb + 1; + uint64_t shift = hb & 63ULL; + uint64_t start = ((1ULL << shift) * first_block_size_) & mask; + size_t pos_in_block = i - start; + auto &block = blocks_[block_i]; + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + + std::unique_lock lock(block.mutex); + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + block.resize(start); + block.is_ready.store(true, std::memory_order_release); + return block.data.at(pos_in_block); + } + template + void push_back(S &&value) { + at(end_.fetch_add(1, std::memory_order_relaxed)) = std::forward(value); + } + T pop_front() { + auto pos = begin_.fetch_add(1, std::memory_order_relaxed); + while (pos >= end_.load(std::memory_order_acquire)) { + // This may (or may not) use too much CPU + td::this_thread::yield(); + } + return std::move(at(pos)); + } + size_t size() const { + return end_.load(); + } + + std::array blocks_; + size_t first_block_size_{0}; + std::atomic begin_{0}; + std::atomic end_{0}; +}; +struct CellInfo; + +class CellDbReaderExt; +struct DynamicBocExtCellExtra { + std::shared_ptr reader; +}; + +class DynamicBocCellLoader { + public: + static td::Result> load_data_cell(const ExtCell &cell, + const DynamicBocExtCellExtra &extra); +}; +using DynamicBocExtCell = ExtCell; + +class CellDbReaderExt : public CellDbReader { + public: + virtual td::Result> load_ext_cell(Ref cell) = 0; +}; + +td::Result> DynamicBocCellLoader::load_data_cell(const DynamicBocExtCell &cell, + const DynamicBocExtCellExtra &extra) { + return extra.reader->load_ext_cell(Ref(&cell)); +} + +#define S(x) \ + td::NamedThreadSafeCounter::CounterRef x { \ + nc.get_counter(#x) \ + } + +struct CacheStats { + td::NamedThreadSafeCounter nc; + S(load_cell_ext); + S(load_cell_ext_cache_hits); + S(load_cell_sync); + S(load_cell_sync_cache_hits); + S(load_cell_async); + S(load_cell_async_cache_hits); + S(ext_cells); + S(ext_cells_load); + S(ext_cells_load_cache_hits); + + S(kv_read_found); + S(kv_read_not_found); + + S(sync_with_db); + S(sync_with_db_only_ref); + S(load_cell_no_cache); +}; + +struct CommitStats { + td::NamedThreadSafeCounter nc; + + S(to_inc); + S(to_dec); + + S(gather_new_cells_calls); + S(gather_new_cells_calls_it); + S(update_parents_calls); + S(update_parents_calls_it); + S(dec_calls); + S(dec_calls_it); + + S(new_cells); + S(new_cells_leaves); + + S(new_cells_loaded_not_in_db); + S(new_cells_loaded_in_db); + S(new_cells_not_in_db_fast); + + S(dec_loaded); + S(dec_to_zero); + + S(changes_loaded); + + // new diff logic + S(diff_zero); + S(diff_full); + S(diff_erase); + S(diff_ref_cnt); + + // old full data logic + S(inc_save); + S(inc_save_full); + S(inc_save_only_ref_cnt); + S(inc_new_cell); + S(inc_just_ref_cnt); + + S(dec_save); + S(dec_save_full); + S(dec_save_only_refcnt); + S(dec_save_erase); + S(dec_erase_cell); + S(dec_just_ref_cnt); +}; + +template +struct AtomicPod { + T load() const { + while (true) { + if (auto res = try_read_stable()) { + return res->second; + } + } + } + + template + std::pair update(F &&f) { + while (true) { + auto res = try_read_stable(); + if (!res) { + continue; + } + auto [before, old_data] = *res; + + auto o_new_data = f(old_data); + if (!o_new_data) { + return {old_data, false}; + } + + if (!lock_.compare_exchange_weak(before, before + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + continue; + } + + data_ = *o_new_data; // relaxed store inside lock + lock_.fetch_add(1, std::memory_order_release); + return {*o_new_data, true}; + } + } + + private: + mutable std::atomic lock_{0}; + T data_{}; + + std::optional> try_read_stable() const { + auto before = lock_.load(std::memory_order_acquire); + if (before % 2 == 1) { + return std::nullopt; + } + T temp = data_; // relaxed read is ok, checked by versioning + auto after = lock_.load(std::memory_order_acquire); + if (after != before) { + return std::nullopt; + } + return std::make_pair(before, temp); + } +}; + +struct InDbInfo { + std::vector parents; + std::atomic pending_children{0}; + std::atomic maybe_in_db{true}; + std::atomic visited_in_gather_new_cells{false}; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const InDbInfo &info) { + sb << "mb_in_db:" << info.maybe_in_db.load() << " chld_n:" << info.pending_children + << " prnt_n:" << info.parents.size(); + return sb; +} +struct CellInfo { + struct State { + // db_ref_cnt and in_db are correct + bool sync_with_db{false}; + + // ignore if sync_with_db is false + td::int32 db_ref_cnt{0}; + td::int32 db_refcnt_fixup{0}; + + // if true - cell is definitely in db + // if false - we know that cell is not in db only is sync_with_db=true + bool in_db{false}; + + // diff to be applied + }; + AtomicPod state; + std::atomic ref_cnt_diff{0}; + + std::atomic visited{false}; + td::unique_ptr in_db_info_ptr; + std::mutex mutex; + + // Could be AtomicRef, but is am not sure that it is worth it + const Ref cell; + + explicit CellInfo(Ref cell) : cell(std::move(cell)) { + } + + InDbInfo &in_db_info() { + return *in_db_info_ptr; + } + const InDbInfo &in_db_info() const { + return *in_db_info_ptr; + } + InDbInfo &in_db_info_create() { // NOT thread safe + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + return in_db_info(); + } + InDbInfo &in_db_info_create(CellInfo *parent) { // Thread Safe + std::unique_lock lock(mutex); + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + auto &res = *in_db_info_ptr; + if (parent != nullptr) { + res.parents.emplace_back(parent); + } + lock.unlock(); + return res; + } + void in_db_info_destroy() { + in_db_info_ptr = nullptr; + } + td::int32 inc_ref_cnt() { + return ref_cnt_diff.fetch_add(1, std::memory_order_relaxed) + 1; + } + td::int32 dec_ref_cnt() { + return ref_cnt_diff.fetch_sub(1, std::memory_order_relaxed) - 1; + } + td::int32 get_ref_cnt_diff() const { + return ref_cnt_diff.load(std::memory_order_relaxed); + } + + void set_not_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.db_ref_cnt == 0); + CHECK(!state.in_db); + return {}; + } + state.sync_with_db = true; + state.in_db = false; + state.db_ref_cnt = 0; + return state; + }); + } + void set_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + //LOG_CHECK(state.in_db) << *this; + return {}; + } + state.in_db = true; + return state; + }); + } + void synced_with_db(td::int32 db_ref_cnt) { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.in_db); + CHECK(state.db_ref_cnt == db_ref_cnt); + return {}; + } + state.in_db = true; + state.db_ref_cnt = db_ref_cnt; + return state; + }); + } + bool visit() { + return !visited.exchange(true); + } + void on_written_to_db() { + auto diff = ref_cnt_diff.exchange(0); + state.update([&](State state) -> std::optional { + if (diff == 0) { + return {}; + } + if (state.sync_with_db) { + state.db_ref_cnt += diff; + CHECK(state.db_ref_cnt >= 0); + state.in_db = state.db_ref_cnt > 0; + } else { + CHECK(diff > 0); + state.in_db = true; + state.db_refcnt_fixup += diff; + } + return state; + }); + } + + td::Result> get_data_cell() { + TRY_RESULT(loaded_cell, cell->load_cell()); + return loaded_cell.data_cell; + } + Cell::Hash key() const { + return cell->get_hash(); + } + bool operator<(const CellInfo &other) const { + return key() < other.key(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const CellInfo &info) { + if (info.cell->is_loaded()) { + auto data_cell = info.cell->load_cell().move_as_ok().data_cell; + vm::CellSlice cs(vm::NoVm{}, data_cell); + sb << data_cell->get_hash().to_hex().substr(0, 8) << " refs:" << data_cell->size_refs() + << " data:" << cs.data_bits().to_hex(cs.size()) << " data_ptr=" << data_cell.get() << " data_ref_cnt(" + << data_cell->get_refcnt() << ")"; + } else { + sb << info.cell->get_hash().to_hex().substr(0, 8); + } + auto state = info.state.load(); + sb << " " << &info; + sb << "\n\tin_db=" << state.in_db << " sync_with_db=" << state.sync_with_db + << " ref_cnt_diff=" << info.get_ref_cnt_diff() << " db_ref_cnt=" << state.db_ref_cnt + << " db_ref_cnt_fixup=" << state.db_refcnt_fixup; + if (state.sync_with_db) { + sb << " REFS(" << info.get_ref_cnt_diff() + state.db_ref_cnt << ")"; + } + if (info.in_db_info_ptr) { + sb << " " << info.in_db_info(); + } + sb << " visited=" << info.visited.load(); + return sb; +} + +struct ExecutorOptions { + size_t extra_threads_n{0}; + std::shared_ptr async_executor; +}; +template +class ExecutorImpl { + public: + ExecutorImpl(ExecutorOptions options) : options_(options) { + } + ExecutorOptions options_; + using InputData = std::vector>; + using OutputData = std::vector>; + struct InputChunk { + td::Span infos; + size_t begin{}; + size_t end{}; + }; + + template + OutputData process(const InputData &data, const F &process_task_f) { + if (options_.extra_threads_n > 0) { + return process_parallel(data, process_task_f); + } else { + return process_sequential(data, process_task_f); + } + } + template + struct SingleThreadWorker { + const F &process_task_f; + mutable std::vector results{}; + void add_task(InputT input) const { + process_task_f(input, *this); + } + void add_result(OutputT output) const { + results.push_back(output); + } + }; + template + OutputData process_sequential(const InputData &data, const F &process_task_f) { + auto w = SingleThreadWorker{process_task_f}; + for (auto &chunk : data) { + for (auto &info : chunk) { + process_task_f(info, w); + } + } + + return {std::move(w.results)}; + } + + template + struct Shared; + + template + struct Worker { + size_t worker_i{}; + std::shared_ptr> shared; + + void add_task(InputT input) const { + shared->delay_or_process_task(input, *this); + } + void add_result(OutputT value) const { + shared->add_result(value, worker_i); + } + void loop() const { + shared->loop(*this); + } + }; + + template + struct Shared { + Shared(size_t workers_n, const InputData &input_data, const ProcessTaskF &process_task_f) + : input_chunks(prepare_input_chunks(input_data)) + , workers_n(workers_n) + , input_size(input_chunks.empty() ? 0 : input_chunks.back().end) + , batch_size(std::clamp(input_size / workers_n / 4, size_t(1), size_t(128))) + , process_task_f(process_task_f) { + } + + const std::vector input_chunks; + + const size_t workers_n{0}; + const size_t input_size{0}; + const size_t batch_size{128}; + + const ProcessTaskF &process_task_f; + + // Position in input + std::atomic next_input_i{0}; + + // Shared queue + // Probably a simpler queue would also work fine + td::MpmcQueue mpmc_queue{workers_n}; + using Waiter = td::MpmcSleepyWaiter; + Waiter waiter; + std::atomic mpmc_queue_size{workers_n}; // guard + + // Output vectors + struct ThreadData { + std::vector output; + char pad[TD_CONCURRENCY_PAD - sizeof(output)]; + }; + std::vector thread_data{workers_n}; + + auto prepare_input_chunks(const InputData &input_data) { + std::vector chunks; + for (auto &chunk : input_data) { + size_t prev_end = chunks.empty() ? 0 : chunks.back().end; + chunks.push_back({.infos = td::as_span(chunk), .begin = prev_end, .end = prev_end + chunk.size()}); + } + return chunks; + } + + void delay_or_process_task(InputT input, const Worker &worker) { + // if there is enough tasks in queue, we continue recursion + if (mpmc_queue_size.load(std::memory_order_acquire) > 256) { + process_task_f(input, worker); + } else { + mpmc_queue_size.fetch_add(1, std::memory_order_acq_rel); + mpmc_queue.push(input, worker.worker_i); + waiter.notify(); + } + } + + void add_result(OutputT result, size_t worker_i) { + thread_data[worker_i].output.push_back(std::move(result)); + } + + void process_initial_input(const Worker &worker) { + size_t input_chunk_i = 0; + while (true) { + auto begin_i = next_input_i.fetch_add(batch_size, std::memory_order_relaxed); + auto end_i = begin_i + batch_size; + if (begin_i >= input_size) { + break; + } + for (size_t i = begin_i; i < end_i && i < input_size; i++) { + while (input_chunks[input_chunk_i].end <= i) { + input_chunk_i++; + } + auto offset = i - input_chunks[input_chunk_i].begin; + auto task = input_chunks[input_chunk_i].infos[offset]; + process_task_f(task, worker); + } + } + } + + void on_processed_task_from_queue(size_t worker_i) { + if (mpmc_queue_size.fetch_add(-1, std::memory_order_acq_rel) == 1) { + for (size_t i = 0; i < workers_n; i++) { + mpmc_queue.push(nullptr, worker_i); + waiter.notify(); + } + } + } + + void process_queue(const Worker &worker) { + on_processed_task_from_queue(worker.worker_i); + + Waiter::Slot slot; + waiter.init_slot(slot, td::narrow_cast(worker.worker_i)); + + while (true) { + InputT input{}; + if (mpmc_queue.try_pop(input, worker.worker_i)) { + waiter.stop_wait(slot); + if (!input) { + break; + } + process_task_f(input, worker); + on_processed_task_from_queue(worker.worker_i); + } else { + waiter.wait(slot); + } + } + } + void loop(const Worker &worker) { + process_initial_input(worker); + process_queue(worker); + } + void finish() const { + CHECK(mpmc_queue_size == 0); + } + }; + + template + OutputData process_parallel(const InputData &input_data, const F &process_task_f) { + const size_t workers_n = options_.extra_threads_n + 1; + auto shared = std::make_shared>(workers_n, input_data, process_task_f); + + CHECK(workers_n >= 1); + for (size_t i = 0; i < workers_n; i++) { + auto to_run = [worker = Worker{.worker_i = i, .shared = shared}] { worker.loop(); }; + + if (i + 1 == workers_n) { + to_run(); + } else if (options_.async_executor) { + options_.async_executor->execute_async(std::move(to_run)); + } else { + // NB: td::thread, NOT std::thread + td::thread(std::move(to_run)).detach(); + } + } + shared->finish(); + return td::transform(shared->thread_data, [](auto &&x) { return std::move(x.output); }); + } +}; +struct Executor { + Executor(ExecutorOptions options = {}) : options_(options) { + } + + template + auto operator()(const std::vector> &data, const F &process_task_f) { + return ExecutorImpl(options_).process(data, process_task_f); + } + + private: + ExecutorOptions options_; +}; + +// Thread safe storage for CellInfo +// Will be used by everybody as shared cache. Yes there is some overhead, but it don't want to create other hash table +struct CellInfoStorage { + public: + // All methods are thead safe + // All CellInfo pointers lives as long as CellInfoStorage + + // returns CellInfo, only if it is already exists + CellInfo *get_cell_info(td::Slice hash) { + return lock(hash)->hash_table.get_if_exists(hash); + } + + CellInfo &create_cell_info_from_db(Ref data_cell, td::int32 ref_cnt) { + auto &info = create_cell_info_from_data_cell(std::move(data_cell)); + info.synced_with_db(ref_cnt); + return info; + } + + // Creates CellInfo from data_cell, or updates existing CellInfo if is not yet loaded + CellInfo &create_cell_info_from_data_cell(Ref cell) { + CHECK(cell.not_null()); + CHECK(cell->is_loaded()); + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + + if (!created) { + info.cell->set_data_cell(std::move(cell)); + } + return info; + } + + // Creates CellInfo from cell. If cell is loaded, it will be used to rewrite or udpate current cell + CellInfo &create_cell_info(Ref cell, CellDbReaderExt *from_reader, CacheStats &stats) { + if (cell->is_loaded()) { + return create_cell_info_from_data_cell(cell->load_cell().move_as_ok().data_cell); + } + + bool our_ext_cell = false; + auto ext_cell = dynamic_cast(cell.get()); + if (ext_cell) { + auto prunned_cell = ext_cell->get_prunned_cell(); + if (prunned_cell.not_null()) { + our_ext_cell = prunned_cell->get_extra().reader.get() == from_reader; + } + our_ext_cell = true; + } else if (!cell->is_loaded()) { + // if we cached cell from OTHER db is good idea to drop it ASAP + force_drop_cache_.store(true, std::memory_order_relaxed); + } + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + if (our_ext_cell) { + stats.ext_cells_load.inc(); + if (info.cell->is_loaded()) { + stats.ext_cells_load_cache_hits.inc(); + } + info.set_in_db(); + } + return info; + } + + void dump() { + LOG(ERROR) << "===========BEGIN DUMP==========="; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + bucket.hash_table.for_each([&](auto &info) { LOG(INFO) << info; }); + } + LOG(ERROR) << "===========END DUMP==========="; + } + + size_t cache_size() { + size_t res = 0; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + res += bucket.hash_table.size(); + } + return res; + } + bool force_drop_cache() { + return force_drop_cache_.load(std::memory_order_relaxed); + } + + private: + struct Bucket { + std::mutex mutex; + CellHashTable hash_table; + }; + constexpr static size_t buckets_n = 8192; + std::array bucket_; + + struct Unlock { + void operator()(Bucket *bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; + std::atomic force_drop_cache_{false}; + + std::unique_ptr lock(Bucket &bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key) { + auto hash = td::as(key.substr(16, 8).ubegin()); + auto bucket_i = hash % buckets_n; + return lock(buckets_[bucket_i]); + } +}; + +class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { + public: + explicit DynamicBagOfCellsDbImplV2(CreateV2Options options) : options_(options) { + get_thread_safe_counter().inc(); + // LOG(ERROR) << "Constructor called for DynamicBagOfCellsDbImplV2"; + } + ~DynamicBagOfCellsDbImplV2() { + // LOG(ERROR) << "Destructor called for DynamicBagOfCellsDbImplV2"; + get_thread_safe_counter().add(-1); + + if (cell_db_reader_) { + cell_db_reader_->drop_cache(); + } + } + + td::Result>> meta_get_all(size_t max_count) const override { + CHECK(meta_db_fixup_.empty()); + std::vector> result; + auto s = cell_db_reader_->key_value_reader().for_each_in_range( + "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + auto it = meta_db_fixup_.find(key); + if (it != meta_db_fixup_.end()) { + if (it->second.empty()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + return cell_db_reader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } + td::Result> load_cell(td::Slice hash) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell(hash); + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result>> load_bulk(td::Span hashes) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_bulk(hashes); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + // TODO: it is better to use AtomicRef, or atomic shared pointer + // But to use AtomicRef we need a little refactoring + // And std::atomic> is still unsupported by clang + std::unique_lock lock(atomic_cell_db_reader_mutex_); + auto reader = atomic_cell_db_reader_; + lock.unlock(); + if (!reader) { + return td::Status::Error("Empty reader"); + } + return reader->load_cell(hash); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell_async(hash, std::move(executor), std::move(promise)); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + auto promise_ptr = std::make_shared>(std::move(promise)); + executor->execute_async([this, promise_ptr = std::move(promise_ptr)] { + prepare_commit(); + promise_ptr->set_value(td::Unit()); + }); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + bool is_prepared_for_commit() { + return to_inc_.empty() && to_dec_.empty(); + } + + Stats get_stats_diff() override { + return {}; + } + + td::Status prepare_commit() override { + if (is_prepared_for_commit()) { + return td::Status::OK(); + } + // NB: we don't use options.executor, because it is prone to deadlocks. We need extra_threads_n threads + // available for blocking + Executor executor{{.extra_threads_n = options_.extra_threads, .async_executor = {}}}; + // calculate in_db for all vertices reachable from to_inc_ roots + // - for ext cells we already know they are in db + // - calculate in_db up from leaves + // - if at least one child is not in db, then the cell is definitely not in db + // - so in best case only leaves will be loaded from db + // - this is optional step. All other logic must work in any case + // - only already loaded cells are loaded from db + + stats_.to_inc.add(to_inc_.size()); + stats_.to_dec.add(to_dec_.size()); + + std::vector> visited_cells; + auto add_visited_cells = [&](std::vector> new_visited_cells) { + for (auto &x : new_visited_cells) { + visited_cells.push_back(std::move(x)); + } + }; + + std::vector> new_cells_leaves; + { + td::PerfWarningTimer timer("celldb_v2: gather_new_cells"); + std::vector prepared_to_inc; + std::vector visited_roots; + for (auto &cell : to_inc_) { + auto &info = cell_db_reader_->cell_info(cell); + if (info.inc_ref_cnt() == 1 && info.visit()) { + visited_roots.push_back(&info); + } + if (info.state.load().in_db) { + continue; + } + auto &in_db_info = info.in_db_info_create(nullptr); + if (!in_db_info.visited_in_gather_new_cells.exchange(true)) { + prepared_to_inc.push_back(&info); + } + } + new_cells_leaves = + executor({std::move(prepared_to_inc)}, [&](CellInfo *info, auto &worker) { gather_new_cells(info, worker); }); + visited_cells.push_back(std::move(visited_roots)); + } + + // LOG(WARNING) << "new_cells_leaves: " << new_cells_leaves.size(); + { + td::PerfWarningTimer timer("celldb_v2: update_parents"); + add_visited_cells( + executor({std::move(new_cells_leaves)}, [&](CellInfo *info, auto &worker) { update_parents(info, worker); })); + } + { + td::PerfWarningTimer timer("dec"); + std::vector prepared_to_dec; + for (auto &cell : to_dec_) { + auto &info = cell_db_reader_->cell_info(cell); + prepared_to_dec.push_back(&info); + } + add_visited_cells( + executor({std::move(prepared_to_dec)}, [&](CellInfo *info, auto &worker) { dec_cell(info, worker); })); + } + + td::PerfWarningTimer timer_serialize("celldb_v2: save_diff_serialize", 0.01); + // LOG(INFO) << "threads_n = " << options_.extra_threads + 1; + diff_chunks_ = executor.operator()( + visited_cells, [&](CellInfo *info, auto &worker) { serialize_diff(info, worker); }); + timer_serialize.reset(); + + { + td::PerfWarningTimer timer("celldb_v2: clear"); + to_inc_.clear(); + to_dec_.clear(); + } + + //cell_db_reader_->dump(); + return td::Status::OK(); + } + + td::Status commit(CellStorer &storer) override { + prepare_commit(); + save_diff(storer); + // We DON'T delete entries from cache, so cache actually represents diff with snapshot in reader + // But we don't want took keep old snapshot forever + LOG_IF(ERROR, dbg) << "clear cell_db_reader"; + //cell_db_reader_->dump(); + //TODO: call drop_cache reliably via rtti + + constexpr bool always_drop_cache = false; + if (always_drop_cache) { + td::PerfWarningTimer timer("celldb_v2: reset reader"); + cell_db_reader_->drop_cache(); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cache_stats_.stats_int["commits"] += 1; + cell_db_reader_ = {}; + // keep atomic reader, to it will be reused + } + return td::Status::OK(); + } + + std::shared_ptr get_cell_db_reader() override { + CHECK(cell_db_reader_); + return cell_db_reader_; + } + + td::Status set_loader(std::unique_ptr loader) override { + if (cell_db_reader_) { + auto cache_size = cell_db_reader_->cache_size(); + bool force_drop_cache = cell_db_reader_->force_drop_cache(); + if (loader && cache_size < options_.cache_size_max && cell_db_reader_ttl_ < options_.cache_ttl_max && + !force_drop_cache) { + // keep cache + cell_db_reader_ttl_++; + return td::Status::OK(); + } + + td::PerfWarningTimer timer(PSTRING() << "celldb_v2: reset reader, TTL=" << cell_db_reader_ttl_ << "/" + << options_.cache_ttl_max << ", cache_size=" << cache_size + << ", force_drop_cache=" << force_drop_cache); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cell_db_reader_->drop_cache(); + cell_db_reader_ = {}; + meta_db_fixup_ = {}; + cell_db_reader_ttl_ = 0; + } + + if (loader) { + cell_db_reader_ = std::make_shared(std::move(loader)); + cell_db_reader_ttl_ = 0; + } + + { + std::lock_guard guard(atomic_cell_db_reader_mutex_); + atomic_cell_db_reader_ = cell_db_reader_; + } + return td::Status::OK(); + } + + void set_celldb_compress_depth(td::uint32 value) override { + celldb_compress_depth_ = value; + } + + vm::ExtCellCreator &as_ext_cell_creator() override { + CHECK(cell_db_reader_); + return *cell_db_reader_; + } + td::Result get_stats() override { + auto ps = stats_.nc.get_stats().with_prefix("storage_"); + ps.apply_diff(cache_stats_.with_prefix("cache_cum_")); + if (cell_db_reader_) { + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_now_")); + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_cum_")); + } + Stats res; + res.named_stats = std::move(ps); + res.named_stats.stats_int["cache.size"] = cell_db_reader_ ? cell_db_reader_->cache_size() : 0; + res.named_stats.stats_int["cache.size_max"] = options_.cache_size_max; + res.named_stats.stats_int["cache.ttl"] = cell_db_reader_ttl_; + res.named_stats.stats_int["cache.ttl_max"] = options_.cache_ttl_max; + return res; + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); + return res; + } + + class CellDbReaderImpl : public CellDbReaderExt, + public ExtCellCreator, + public std::enable_shared_from_this { + public: + explicit CellDbReaderImpl(std::unique_ptr cell_loader) : cell_loader_(std::move(cell_loader)) { + } + + size_t cache_size() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->cache_size(); + } + return 0; + } + bool force_drop_cache() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->force_drop_cache(); + } + return false; + } + void drop_cache() { + // NOT thread safe + internal_storage_.reset(); + } + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + // thread safe function + stats_.ext_cells.inc(); + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{shared_from_this()})); + + return ext_cell; + } + CellInfo *register_ext_cell_inner(Ref ext_cell, CellInfoStorage &storage) { + auto &info = storage.create_cell_info(std::move(ext_cell), this, stats_); + return &info; + } + + void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) { + // thread safe function + stats_.load_cell_async.inc(); + auto maybe_cell = load_cell_fast_path(hash, false, nullptr); + if (maybe_cell.not_null()) { + stats_.load_cell_async_cache_hits.inc(); + return promise.set_value(std::move(maybe_cell)); + } + auto promise_ptr = std::make_shared>>(std::move(promise)); + + executor->execute_async( + [self = shared_from_this(), promise_ptr = std::move(promise_ptr), hash = CellHash::from_slice(hash)]() { + promise_ptr->set_result(self->load_cell(hash.as_slice())); + }); + } + + td::Result> load_cell(td::Slice hash) override { + // thread safe function + stats_.load_cell_sync.inc(); + bool loaded{false}; + auto maybe_cell = load_cell_fast_path(hash, true, &loaded); + if (maybe_cell.not_null()) { + if (!loaded) { + stats_.load_cell_sync_cache_hits.inc(); + } + return maybe_cell; + } + return load_cell_slow_path(hash); + } + + td::Result>> load_bulk(td::Span hashes) override { + // thread safe function + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto maybe_cell = load_cell(hash); + if (maybe_cell.is_error()) { + return maybe_cell.move_as_error(); + } + result.push_back(maybe_cell.move_as_ok()); + } + return result; + } + + td::Result> load_ext_cell(Ref ext_cell) override { + // thread safe function. + // Called by external cell + stats_.load_cell_ext.inc(); + auto storage = weak_storage_.lock(); + if (!storage) { + TRY_RESULT(load_result, load_cell_no_cache(ext_cell->get_hash().as_slice())); + return load_result.cell_; + } + // we delayed registering ext cell till this moment + auto cell_info = register_ext_cell_inner(std::move(ext_cell), *storage); + + CHECK(cell_info != nullptr); // currently all ext_cells are registered in cache + if (!cell_info->cell->is_loaded()) { + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); // critical, better to fail + } else { + stats_.load_cell_ext_cache_hits.inc(); + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + + CellInfo &cell_info(Ref cell) { + // thread safe function, but called only by DB + CHECK(internal_storage_) + return internal_storage_->create_cell_info(std::move(cell), this, stats_); + } + + std::pair sync_with_db(CellInfo &info, bool need_data) { + // thread safe function, but called only by DB + auto effective_need_data = need_data; + if (info.cell->is_loaded()) { + effective_need_data = false; + } + return info.state.update([&](CellInfo::State state) -> std::optional { + if (state.sync_with_db) { + return {}; + } + stats_.sync_with_db.inc(); + if (!effective_need_data) { + stats_.sync_with_db_only_ref.inc(); + } + auto load_result = + cell_loader_->load(info.cell->get_hash().as_slice(), effective_need_data, *this).move_as_ok(); + + state.sync_with_db = true; + if (load_result.status == CellLoader::LoadResult::NotFound) { + CHECK(state.in_db == false); + CHECK(state.db_ref_cnt == 0); + stats_.kv_read_not_found.inc(); + return state; + } + stats_.kv_read_found.inc(); + + state.in_db = true; + state.db_ref_cnt = load_result.refcnt() + state.db_refcnt_fixup; + if (load_result.cell().not_null()) { + info.cell->set_data_cell(std::move(load_result.cell())); + } + CHECK(!need_data || info.cell->is_loaded()); + return state; + }); + } + + void dump() { + internal_storage_->dump(); + } + + td::NamedStats get_stats() const { + return stats_.nc.get_stats(); + } + td::KeyValueReader &key_value_reader() { + return cell_loader_->key_value_reader(); + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); + return res; + } + std::shared_ptr internal_storage_{std::make_shared()}; + std::weak_ptr weak_storage_{internal_storage_}; + std::unique_ptr cell_loader_; + CacheStats stats_; + + Ref load_cell_fast_path(td::Slice hash, bool may_block, bool *loaded) { + auto storage = weak_storage_.lock(); + if (!storage) { + return {}; + } + auto cell_info = storage->get_cell_info(hash); + if (cell_info != nullptr) { + if (!cell_info->cell->is_loaded()) { + if (may_block) { + if (loaded) { + *loaded = true; + } + CHECK(cell_info->state.load().in_db); + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); + } else { + return {}; + } + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + return {}; + } + td::Result load_cell_no_cache(td::Slice hash) { + stats_.load_cell_no_cache.inc(); + TRY_RESULT(load_result, cell_loader_->load(hash, true, *this)); + if (load_result.status == CellLoader::LoadResult::NotFound) { + stats_.kv_read_not_found.inc(); + return td::Status::Error("Cell load failed: not in db"); + } + stats_.kv_read_found.inc(); + return load_result; + } + td::Result> load_cell_slow_path(td::Slice hash) { + TRY_RESULT(load_result, load_cell_no_cache(hash)); + auto storage = weak_storage_.lock(); + if (!storage) { + return load_result.cell_; + } + auto &cell_info = storage->create_cell_info_from_db(std::move(load_result.cell()), load_result.refcnt()); + return cell_info.cell->load_cell().move_as_ok().data_cell; + } + }; + + CreateV2Options options_; + td::int32 celldb_compress_depth_{0}; + std::vector> to_inc_; + std::vector> to_dec_; + std::vector> diff_chunks_; + std::vector meta_diffs_; + std::map> meta_db_fixup_; + + mutable std::mutex atomic_cell_db_reader_mutex_; + std::shared_ptr atomic_cell_db_reader_; + + std::shared_ptr cell_db_reader_; + size_t cell_db_reader_ttl_{0}; + td::NamedStats cache_stats_; + CommitStats stats_; + bool dbg{false}; + + template + void gather_new_cells(CellInfo *info, WorkerT &worker) { + stats_.gather_new_cells_calls.inc(); + do { + // invariant: info is not in DB; with created in_db_info + // we enter into each root only once + stats_.gather_new_cells_calls_it.inc(); + stats_.new_cells.inc(); + auto &in_db_info = info->in_db_info(); + + CellSlice cs(vm::NoVm{}, info->cell); // ensure cell is loaded + CellInfo *prev_child_info = nullptr; + while (cs.have_refs()) { + auto *child_info = &cell_db_reader_->cell_info(cs.fetch_ref()); + auto child_state = child_info->state.load(); + + if (child_state.in_db) { + LOG_IF(INFO, dbg) << "gather_new_cells: IN DB\n\tchld: " << *child_info; + continue; + } + + auto &child_in_db_info = child_info->in_db_info_create(info); + in_db_info.pending_children.fetch_add(1, std::memory_order_relaxed); + + if (child_in_db_info.visited_in_gather_new_cells.exchange(true)) { + continue; + } + + if (prev_child_info != nullptr) { + worker.add_task(prev_child_info); + } + prev_child_info = child_info; + } + LOG_IF(INFO, dbg) << "gather_new_cells: NOT IN DB\n\t" << *info; + if (in_db_info.pending_children.load(std::memory_order_relaxed) == 0) { + worker.add_result(info); + stats_.new_cells_leaves.inc(); + LOG_IF(WARNING, dbg) << "gather_new_cells: ADD LEAVE\n\t" << *info; + } + info = prev_child_info; + } while (info != nullptr); + } + + template + void update_parents(CellInfo *info, const WorkerT &worker) { + stats_.update_parents_calls.inc(); + size_t it = 0; + do { + stats_.update_parents_calls_it.inc(); + it++; + //LOG(INFO) << "update_parents: it=" << it << "\n\t"; + auto &in_db_info = info->in_db_info(); + bool in_db = false; + if (in_db_info.maybe_in_db.load(std::memory_order_relaxed)) { + auto [state, loaded] = cell_db_reader_->sync_with_db(*info, false); + in_db = state.in_db; + if (in_db) { + stats_.new_cells_loaded_in_db.inc(); + } else { + stats_.new_cells_loaded_not_in_db.inc(); + } + } else { + stats_.new_cells_not_in_db_fast.inc(); + info->set_not_in_db(); + } + LOG_IF(INFO, dbg) << "update_parents: it=" << it << "\n\t" << *info; + + CellInfo *prev_parent{nullptr}; + for (auto &parent : in_db_info.parents) { + auto &parent_in_db_info = parent->in_db_info(); + if (!in_db) { + parent_in_db_info.maybe_in_db.store(false, std::memory_order_relaxed); + } + if (parent_in_db_info.pending_children.fetch_sub(1, std::memory_order_release) == 1) { + if (prev_parent) { + worker.add_task(prev_parent); + } + prev_parent = parent; + } + } + if (!in_db) { + CellSlice cs(vm::NoVm{}, info->cell); + while (cs.have_refs()) { + auto child = cs.fetch_ref(); + auto &child_info = cell_db_reader_->cell_info(std::move(child)); + if (child_info.inc_ref_cnt() == 1 && child_info.visit()) { + worker.add_result(&child_info); + } + } + } + info->in_db_info_destroy(); + info = prev_parent; + } while (info); + } + + template + void dec_cell(CellInfo *info, WorkerT &worker) { + stats_.dec_calls.inc(); + + while (true) { + stats_.dec_calls_it.inc(); + if (info->visit()) { + worker.add_result(info); + } + auto ref_cnt_diff = info->dec_ref_cnt(); + if (ref_cnt_diff > 0) { + LOG_IF(INFO, dbg) << "NOT DEC" + << "\n\t" << info; + break; + } + auto state = info->state.load(); + if (ref_cnt_diff == 0 && state.in_db) { + LOG_IF(INFO, dbg) << "NOT DEC (in_db) " + << "\n\t" << info; + break; + } + if (!state.sync_with_db) { + state = cell_db_reader_->sync_with_db(*info, true).first; + stats_.dec_loaded.inc(); + CHECK(ref_cnt_diff == 0 || state.in_db); + } + auto ref_cnt = state.db_ref_cnt + ref_cnt_diff; + if (ref_cnt > 0) { + LOG_IF(INFO, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } else { + LOG_IF(ERROR, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } + CHECK(ref_cnt >= 0); + if (ref_cnt > 0) { + break; + } + stats_.dec_to_zero.inc(); + CellSlice cs(vm::NoVm{}, info->cell); + if (!cs.have_refs()) { + break; + } + while (cs.size_refs() > 1) { + worker.add_task(&cell_db_reader_->cell_info(cs.fetch_ref())); + } + info = &cell_db_reader_->cell_info(cs.fetch_ref()); + } + } + + template + void serialize_diff(CellInfo *info, Worker &worker) { + info->visited.store(false, std::memory_order_relaxed); + auto ref_cnt_diff = info->get_ref_cnt_diff(); + if (ref_cnt_diff == 0) { + stats_.diff_zero.inc(); + return; + } + auto should_compress = celldb_compress_depth_ != 0 && info->cell->get_depth() == celldb_compress_depth_; + + bool merge_supported = true; + if (merge_supported) { + auto state = info->state.load(); + if (ref_cnt_diff < 0) { + CHECK(state.sync_with_db); + /* + if (state.db_ref_cnt + ref_cnt_diff == 0) { + LOG(ERROR) << "DEC ERASE " << info->cell->get_hash().to_hex(); + } else { + LOG(ERROR) << "DEC MERGE " << info->cell->get_hash().to_hex() << *info; + } + */ + } + if (ref_cnt_diff < 0 && state.sync_with_db && state.db_ref_cnt + ref_cnt_diff == 0) { + // Erase is better than Merge+CompactionFilter + // So I see no reason for CompactionFilter at all + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.diff_erase.inc(); + } else { + bool with_data = ref_cnt_diff > 0 && !state.in_db; + if (with_data) { + CHECK(state.sync_with_db); + auto data_cell = info->cell->load_cell().move_as_ok().data_cell; + stats_.diff_full.inc(); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, should_compress)}); + } else { + stats_.diff_ref_cnt.inc(); + worker.add_result({.type = CellStorer::Diff::Merge, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_refcnt_diffs(ref_cnt_diff)}); + } + } + info->on_written_to_db(); + return; + } + + auto state = info->state.load(); + if (!state.sync_with_db) { + stats_.changes_loaded.inc(); + state = cell_db_reader_->sync_with_db(*info, true).first; + } + CHECK(state.sync_with_db); + auto new_ref_cnt = ref_cnt_diff + state.db_ref_cnt; + + if (ref_cnt_diff < 0) { + stats_.dec_save.inc(); + if (new_ref_cnt == 0) { + stats_.dec_erase_cell.inc(); + + LOG_IF(ERROR, dbg) << "DEC ERASE " << *info; + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.dec_save_erase.inc(); + } else { + stats_.dec_just_ref_cnt.inc(); + + LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; + CHECK(info->cell->is_loaded()); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); + stats_.dec_save_full.inc(); + } + } else { + stats_.inc_save.inc(); + CHECK(info->cell->is_loaded()); + if (state.db_ref_cnt == 0) { + stats_.inc_new_cell.inc(); + LOG_IF(ERROR, dbg) << "INC CREATE " << *info; + } else { + stats_.inc_just_ref_cnt.inc(); + LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; + } + + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); + stats_.inc_save_full.inc(); + } + } + + void save_diff(CellStorer &storer) { + td::PerfWarningTimer timer("celldb_v2: save_diff"); + td::PerfWarningTimer timer_store_to_db("celldb_v2: save_diff_store_to_db", 0.01); + // Have no idea hot to parallelize this in case of rocksdb + for (auto &diffs : diff_chunks_) { + for (auto &diff : diffs) { + storer.apply_diff(diff).ensure(); + } + } + for (auto &meta_diff : meta_diffs_) { + meta_db_fixup_[meta_diff.key] = meta_diff.value; + storer.apply_meta_diff(meta_diff).ensure(); + } + timer_store_to_db.reset(); + td::PerfWarningTimer timer_clear("celldb_v2: save_diff_clear"); + diff_chunks_.clear(); + meta_diffs_.clear(); + timer_clear.reset(); + } +}; +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_v2(CreateV2Options options) { + return std::make_unique(options); +} +} // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 03cad0934..829ed38d8 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -157,17 +157,9 @@ class ArenaPrunnedCellCreator : public ExtCellCreator { } }; - struct Allocator { - template - std::unique_ptr> make_unique(ArgsT &&...args) { - auto *ptr = arena_.alloc(sizeof(T)); - T *obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } - }; td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { - Allocator allocator; - TRY_RESULT(cell, PrunnedCell::create(allocator, PrunnedCellInfo{level_mask, hash, depth}, Counter())); + TRY_RESULT(cell, PrunnedCell::create([&](size_t bytes) { return arena_.alloc(bytes); }, false, + PrunnedCellInfo{level_mask, hash, depth}, Counter())); return cell; } static td::int64 count() { @@ -413,6 +405,7 @@ class CellStorage { size_t dense_ht_size = 0; size_t new_ht_size = 0; for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + // TODO: this leads to CE when use_dense_hash_map == false dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); dense_ht_size += bucket.infos_.dense_ht_size_; new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); @@ -461,6 +454,16 @@ class CellStorage { return td::Status::Error("not found"); } + td::Result>> load_bulk(td::Span hashes) const { + std::vector> res; + res.reserve(hashes.size()); + for (auto &hash : hashes) { + TRY_RESULT(cell, load_cell(hash)); + res.push_back(std::move(cell)); + } + return res; + } + td::Result> load_root_local(const CellHash &hash) const { auto lock = local_access_.lock(); if (auto it = local_roots_.find(hash); it != local_roots_.end()) { @@ -468,6 +471,14 @@ class CellStorage { } return td::Status::Error("not found"); } + td::Result>> load_known_roots_local() const { + auto lock = local_access_.lock(); + std::vector> result; + for (auto &root : roots_) { + result.emplace_back(root); + } + return result; + } td::Result> load_root_shared(const CellHash &hash) const { std::lock_guard lock(root_mutex_); if (auto it = roots_.find(hash); it != roots_.end()) { @@ -592,7 +603,7 @@ class CellStorage { std::vector bucket_stats(buckets_.size()); std::atomic boc_count{0}; for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { - bucket_stats[bucket_id] = validate_bucket_a(bucket, options.use_arena); + bucket_stats[bucket_id] = validate_bucket_a(bucket); boc_count += bucket.boc_count_; }); for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { validate_bucket_b(bucket); }); @@ -620,7 +631,7 @@ class CellStorage { sb << "\n\t" << key << "=" << value; } LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) - << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + << "desc<> keys count is " << desc_count << " which is different from roots count " << stats.roots_total_count; LOG_IF(WARNING, verbose) << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" << desc_count << "\n\tcells_count=" << stats.cells_total_count @@ -721,12 +732,12 @@ class CellStorage { }); } - DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket, bool use_arena) { + DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket) { DynamicBagOfCellsDb::Stats stats; bucket.infos_.for_each([&](auto &it) { int cell_ref_cnt = it.cell->get_refcnt(); - CHECK(it.db_refcnt + 1 + use_arena >= cell_ref_cnt); - auto extra_refcnt = it.db_refcnt + 1 + use_arena - cell_ref_cnt; + CHECK(it.db_refcnt + 1 >= cell_ref_cnt); + auto extra_refcnt = it.db_refcnt + 1 - cell_ref_cnt; if (extra_refcnt != 0) { bucket.roots_.push_back(it.cell); stats.roots_total_count++; @@ -757,18 +768,90 @@ class CellStorage { } }; +class MetaStorage { + public: + explicit MetaStorage(std::vector> values) + : meta_(std::move_iterator(values.begin()), std::move_iterator(values.end())) { + for (auto &p : meta_) { + CHECK(p.first.size() != CellTraits::hash_bytes); + } + } + std::vector> meta_get_all(size_t max_count) const { + std::vector> res; + for (const auto &[k, v] : meta_) { + if (res.size() >= max_count) { + break; + } + res.emplace_back(k, v); + } + return res; + } + KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { + auto lock = local_access_.lock(); + auto it = meta_.find(key.str()); + if (it == meta_.end()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + void meta_set(td::Slice key, td::Slice value) { + auto lock = local_access_.lock(); + meta_[key.str()] = value.str(); + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + } + void meta_erase(td::Slice key) { + auto lock = local_access_.lock(); + meta_.erase(key.str()); + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + } + std::vector extract_diffs() { + auto lock = local_access_.lock(); + return std::move(meta_diffs_); + } + + private: + mutable UniqueAccess local_access_; + std::unordered_map meta_; + std::vector meta_diffs_; +}; + class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { public: - explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + explicit InMemoryBagOfCellsDb(td::unique_ptr storage, td::unique_ptr meta_storage) + : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { + } + + td::Result>> meta_get_all(size_t max_count) const override { + return meta_storage_->meta_get_all(max_count); + } + td::Result meta_get(td::Slice key, std::string &value) override { + CHECK(key.size() != CellTraits::hash_bytes); + return meta_storage_->meta_get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_storage_->meta_set(key, value); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_storage_->meta_erase(key); + return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { return storage_->load_cell(CellHash::from_slice(hash)); } + td::Result>> load_known_roots() const override { + return storage_->load_known_roots_local(); + } td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } + td::Result>> load_bulk(td::Span hashes) override { + return storage_->load_bulk(td::transform(hashes, [](auto &hash) { return CellHash::from_slice(hash); })); + } td::Result> load_root_thread_safe(td::Slice hash) const override { return storage_->load_root_shared(CellHash::from_slice(hash)); } @@ -798,29 +881,37 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { TRY_STATUS(prepare_commit()); } + td::PerfWarningTimer times_save_diff("save diff"); Stats diff; CHECK(to_dec_.empty()); - for (auto &it : info_) { - auto &info = it.second; + for (auto &info : info_) { if (info.diff_refcnt == 0) { continue; } auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); - CHECK(refcnt >= 0); + LOG_CHECK(refcnt >= 0) << info.db_refcnt << " + " << info.diff_refcnt; if (refcnt > 0) { - cell_storer.set(refcnt, info.cell, false); + if (info.db_refcnt == 0) { + TRY_STATUS(cell_storer.set(refcnt, info.cell, false)); + } else { + TRY_STATUS(cell_storer.merge(info.cell->get_hash().as_slice(), info.diff_refcnt)); + } storage_->set(refcnt, info.cell); if (info.db_refcnt == 0) { diff.cells_total_count++; diff.cells_total_size += static_cast(info.cell->get_storage_size()); } } else { - cell_storer.erase(info.cell->get_hash().as_slice()); + TRY_STATUS(cell_storer.erase(info.cell->get_hash().as_slice())); storage_->erase(info.cell->get_hash()); diff.cells_total_count--; diff.cells_total_size -= static_cast(info.cell->get_storage_size()); } } + auto meta_diffs = meta_storage_->extract_diffs(); + for (const auto &meta_diff : meta_diffs) { + TRY_STATUS(cell_storer.apply_meta_diff(meta_diff)); + } storage_->apply_stats_diff(diff); info_ = {}; return td::Status::OK(); @@ -872,13 +963,39 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { private: td::unique_ptr storage_; + td::unique_ptr meta_storage_; struct Info { - td::int32 db_refcnt{0}; - td::int32 diff_refcnt{0}; + mutable td::int32 db_refcnt{0}; + mutable td::int32 diff_refcnt{0}; Ref cell; + vm::CellHash key() const { + return cell->get_hash(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Info &info, const Info &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Info &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Info &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Info &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; }; - td::HashMap info_; + td::HashSet info_; std::unique_ptr loader_; std::vector> to_inc_; @@ -886,13 +1003,13 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { Ref do_inc(Ref cell) { auto cell_hash = cell->get_hash(); - if (auto it = info_.find(cell_hash); it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::max()); - it->second.diff_refcnt++; - return it->second.cell; + if (auto it = info_.find(cell_hash.as_slice()); it != info_.end()) { + CHECK(it->diff_refcnt != std::numeric_limits::max()); + it->diff_refcnt++; + return it->cell; } if (auto o_info = storage_->get_info(cell_hash)) { - info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + info_.emplace(Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); return std::move(o_info->cell); } @@ -905,21 +1022,21 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { } auto res = cb.finalize(cs.is_special()); CHECK(res->get_hash() == cell_hash); - info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + info_.emplace(Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); return res; } void do_dec(Ref cell) { auto cell_hash = cell->get_hash(); - auto it = info_.find(cell_hash); + auto it = info_.find(cell_hash.as_slice()); if (it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::min()); - --it->second.diff_refcnt; + CHECK(it->diff_refcnt != std::numeric_limits::min()); + --it->diff_refcnt; } else { auto info = *storage_->get_info(cell_hash); - it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + it = info_.emplace(Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; } - if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + if (it->diff_refcnt + it->db_refcnt != 0) { return; } CellSlice cs(NoVm{}, std::move(cell)); @@ -936,7 +1053,8 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K if (kv == nullptr) { LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); - return std::make_unique(std::move(storage)); + auto meta_storage = td::make_unique(std::vector>{}); + return std::make_unique(std::move(storage), std::move(meta_storage)); } std::vector keys; @@ -962,6 +1080,9 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K local_desc_count++; return td::Status::OK(); } + if (key.size() != 32) { + return td::Status::OK(); + } auto r_res = CellLoader::load(key, value.str(), true, pc_creator); if (r_res.is_error()) { LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); @@ -983,6 +1104,24 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K }; auto storage = CellStorage::build(options, parallel_scan_cells); - return std::make_unique(std::move(storage)); + + std::vector> meta; + // NB: it scans 1/(2^32) of the database which is not much + kv->for_each_in_range("desc", "desd", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + // this is for tests mostly. desc* keys are expected to correspond to roots + kv->for_each_in_range("meta", "metb", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + auto meta_storage = td::make_unique(std::move(meta)); + + return std::make_unique(std::move(storage), std::move(meta_storage)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 80dbfbf0b..215dcfa3b 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -18,7 +18,6 @@ */ #include "vm/db/StaticBagOfCellsDb.h" -#include "vm/cells/CellWithStorage.h" #include "vm/boc.h" #include "vm/cells/ExtCell.h" @@ -40,6 +39,9 @@ class RootCell : public Cell { struct PrivateTag {}; public: + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { return cell_->load_cell(); } @@ -94,11 +96,11 @@ class DataCellCacheNoop { class DataCellCacheMutex { public: Ref store(int idx, Ref cell) { - auto lock = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); return cells_.emplace(idx, std::move(cell)).first->second; } Ref load(int idx) { - auto lock = cells_rw_mutex_.lock_read(); + std::lock_guard lock(mutex_); auto it = cells_.find(idx); if (it != cells_.end()) { return it->second; @@ -106,12 +108,13 @@ class DataCellCacheMutex { return {}; } void clear() { - auto guard = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); cells_.clear(); } private: - td::RwMutex cells_rw_mutex_; + std::mutex mutex_; + // NB: in case of high contention, one should use multiple buckets with per bucket mutexes td::HashMap> cells_; }; @@ -246,7 +249,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { BagOfCells::Info info_; std::mutex index_i_mutex_; - td::RwMutex index_data_rw_mutex_; + std::mutex index_mutex_; std::string index_data_; std::atomic index_i_{0}; size_t index_offset_{0}; @@ -319,7 +322,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { - guard = index_data_rw_mutex_.lock_read().move_as_ok(); + std::lock_guard guard(index_mutex_); offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } @@ -432,7 +435,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); - auto guard = index_data_rw_mutex_.lock_write(); + std::lock_guard guard(index_mutex_); index_data_.append(reinterpret_cast(tmp), info_.offset_byte_size); } return td::Status::OK(); diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 41f9c3396..0450932a2 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1772,6 +1772,260 @@ void Dictionary::map(const simple_map_func_t& simple_map_func) { map(map_func); } +bool Dictionary::multiset(td::MutableSpan>> new_values) { + force_validate(); + auto cmp = [&](const std::pair>& a, + const std::pair>& b) { + return td::bitstring::bits_memcmp(a.first, b.first, key_bits) < 0; + }; + if (!std::is_sorted(new_values.begin(), new_values.end(), cmp)) { + std::sort(new_values.begin(), new_values.end(), cmp); + } + for (size_t i = 0; i + 1 < new_values.size(); ++i) { + if (td::bitstring::bits_memcmp(new_values[i].first, new_values[i + 1].first, key_bits) == 0) { + return false; + } + } + unsigned char key_buffer[max_key_bytes]; + try { + Ref root = dict_multiset(get_root_cell(), new_values, key_buffer, key_bits, key_bits, 0); + set_root_cell(std::move(root)); + return true; + } catch (CombineError) { + return false; + } +} + +static Ref dict_build(td::Span>> values, int total_key_len, + int prefix_len) { + if (values.empty()) { + return {}; + } + if (values.size() == 1) { + if (values[0].second.is_null()) { + throw CombineError{}; + } + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, total_key_len - prefix_len, total_key_len - prefix_len); + if (!cb.append_builder_bool(values[0].second)) { + throw VmError{Excno::cell_ov, "cannot store new value into a dictionary cell"}; + } + return cb.finalize(); + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values.front().first + prefix_len, values.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + CHECK(prefix_len + common_prefix_len < total_key_len); + size_t idx = 0; + while (values[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + Ref left = dict_build(values.substr(0, idx), total_key_len, prefix_len + common_prefix_len + 1); + Ref right = dict_build(values.substr(idx), total_key_len, prefix_len + common_prefix_len + 1); + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, common_prefix_len, total_key_len - prefix_len); + if (!(cb.store_ref_bool(std::move(left)) && cb.store_ref_bool(std::move(right)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); +} + +// Based on DictionaryFixed::dict_combine_with, but dict2 is replaced with a list of values, mode is false +Ref Dictionary::dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1) { + int prefix_len = total_key_len - n; + for (auto& [k, _] : values2) { + CHECK(td::bitstring::bits_memcmp(k, key_buffer - prefix_len, prefix_len) == 0); + } + if (dict1.is_null()) { + return dict_build(values2, total_key_len, prefix_len); + } + if (values2.empty()) { + assert(!skip1); + return dict1; + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values2.front().first + prefix_len, values2.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + assert(prefix_len + common_prefix_len < total_key_len || values2.size() == 1); + // both dictionaries non-empty + // skip1: remove that much first bits from all keys in dictionary dict1 (its keys are actually n + skip1 bits long) + // resulting dictionary will have n-bit keys + LabelParser label1{dict1, n + skip1, LabelParser::chk_all}; + int l1 = label1.l_bits - skip1, l2 = (int)common_prefix_len; + assert(l1 >= 0 && l2 >= 0); + assert(!skip1 || label1.common_prefix_len(key_buffer - skip1, skip1) == skip1); + int c = label1.common_prefix_len(values2[0].first + prefix_len - skip1, skip1 + l2) - skip1; + label1.extract_label_to(key_buffer - skip1); + assert(c >= 0 && c <= l1 && c <= l2); + if (c < l1 && c < l2) { + // the two dictionaries have disjoint keys + CellBuilder cb; + append_dict_label(cb, key_buffer + c + 1, l1 - c - 1, n - c - 1); + if (!cell_builder_add_slice_bool(cb, *label1.remainder)) { + throw VmError{Excno::cell_ov, "cannot prune label of an old dictionary cell while merging dictionaries"}; + } + label1.remainder.clear(); + dict1 = cb.finalize(); + // cb.reset(); // included into finalize(); + // now dict1 has been "pruned" -- first skip1+c+1 bits removed from its root egde label + Ref dict2 = dict_build(values2, total_key_len, prefix_len + c + 1); + if (!values2[0].first[prefix_len + c]) { + std::swap(dict1, dict2); + } + // put dict1 into the left tree (with smaller labels), dict2 into the right tree + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(dict1)) && cb.store_ref_bool(std::move(dict2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + + auto combine_func = [&](CellBuilder& cb, const Ref& cb2) -> bool { + if (cb2.is_null()) { + return false; + } + cb.append_builder(*cb2); + return true; + }; + size_t idx = 0; + while (prefix_len + common_prefix_len < total_key_len && idx < values2.size() && + values2[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + auto values2_left = values2.substr(0, idx); + auto values2_right = values2.substr(idx); + + if (c == l1 && c == l2) { + // funny enough, the non-skipped parts of labels of l1 and l2 match + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (c == n) { + // our two dictionaries are in fact leafs with matching edge labels (keys) + if (!combine_func(cb, values2[0].second)) { + // alas, the two values did not combine, this key will be absent from resulting dictionary + return {}; + } + return cb.finalize(); + } + assert(c < n); + key_buffer += c + 1; + key_buffer[-1] = 0; + // combine left subtrees + auto c1 = dict_multiset(label1.remainder->prefetch_ref(0), values2_left, key_buffer, n - c - 1, total_key_len, 0); + key_buffer[-1] = 1; + // combine right subtrees + auto c2 = dict_multiset(label1.remainder->prefetch_ref(1), values2_right, key_buffer, n - c - 1, total_key_len, 0); + label1.remainder.clear(); + // c1 and c2 are merged left and right children of dict1 and dict2 + if (!c1.is_null() && !c2.is_null()) { + // both children non-empty, simply put them into the new node + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + if (c1.is_null() && c2.is_null()) { + return {}; // both children empty, resulting dictionary also empty + } + // exactly one of c1 and c2 is non-empty, have to merge labels + bool sw = c1.is_null(); + key_buffer[-1] = sw; + if (sw) { + c1 = std::move(c2); + } + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer); + key_buffer -= c + 1; + // store combined label for the new edge + cb.reset(); + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } + if (c == l1) { + assert(c < l2); + dict1.clear(); + // children of root node of dict1 + auto c1 = label1.remainder->prefetch_ref(0); + auto c2 = label1.remainder->prefetch_ref(1); + label1.remainder.clear(); + // have to merge dict2 with one of the children of dict1 + td::bitstring::bits_memcpy(key_buffer, values2[0].first + prefix_len, l2); // dict2 has longer label, extract it + bool sw = key_buffer[c]; + if (!sw) { + // merge c1 with dict2 + c1 = dict_multiset(std::move(c1), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } else { + // merge c2 with dict2 + c2 = dict_multiset(std::move(c2), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } else { + assert(c == l2 && c < l1); + // have to merge dict1 with one of the children of dict2 + bool sw = key_buffer[c]; + Ref c1, c2; + if (!sw) { + // merge dict1 with c1 + c1 = dict_multiset(std::move(dict1), values2_left, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c2 = dict_build(values2_right, total_key_len, prefix_len + l2 + 1); + } else { + // merge dict1 with c2 + c2 = dict_multiset(std::move(dict1), values2_right, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c1 = dict_build(values2_left, total_key_len, prefix_len + l2 + 1); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } +} + // mode: +1 = forbid empty dict1 with non-empty dict2 // +2 = forbid empty dict2 with non-empty dict1 Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, @@ -2536,6 +2790,12 @@ Ref AugmentedDictionary::get_root() const { return root; } +Ref AugmentedDictionary::get_wrapped_dict_root() const { + vm::CellBuilder cb; + cb.append_cellslice(get_root()); + return cb.finalize(); +} + Ref AugmentedDictionary::extract_root() && { if (!(flags & f_root_cached) && !compute_root()) { return {}; diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index c4044963f..4da56acd9 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -527,13 +527,15 @@ class Dictionary final : public DictionaryFixed { auto range(bool rev = false, bool sgnd = false) { return dict_range(*this, rev, sgnd); } + bool multiset(td::MutableSpan>> new_values); private: bool check_fork(CellSlice& cs, Ref c1, Ref c2, int n) const override { return cs.empty_ext(); } static Ref extract_value_ref(Ref cs); - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + static Ref dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1); }; class PrefixDictionary final : public DictionaryBase { @@ -570,6 +572,7 @@ class AugmentedDictionary final : public DictionaryFixed { AugmentedDictionary(DictNonEmpty, Ref _root, int _n, const AugmentationData& _aug, bool validate = true); Ref get_empty_dictionary() const; Ref get_root() const; + Ref get_wrapped_dict_root() const; Ref extract_root() &&; bool append_dict_to_bool(CellBuilder& cb) &&; bool append_dict_to_bool(CellBuilder& cb) const &; diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index d209c88ed..f6e35c6fe 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -32,6 +32,7 @@ namespace { class LargeBocSerializer { public: using Hash = Cell::Hash; + constexpr static int load_batch_size = 4'000'000; explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { } @@ -46,7 +47,6 @@ class LargeBocSerializer { private: std::shared_ptr reader; struct CellInfo { - Cell::Hash hash; std::array ref_idx; int idx; unsigned short serialized_size; @@ -95,6 +95,8 @@ void LargeBocSerializer::add_root(Hash root) { roots.emplace_back(root, -1); } +// Unlike crypto/vm/boc.cpp this implementation does not load all cells into memory +// and traverses them in BFS order to utilize bulk load of cells on the same level. td::Status LargeBocSerializer::import_cells() { if (logger_ptr_) { logger_ptr_->start_stage("import_cells"); @@ -111,46 +113,124 @@ td::Status LargeBocSerializer::import_cells() { return td::Status::OK(); } -td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { - if (depth > Cell::max_depth) { - return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); +td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) { + const int start_ind = cell_count; + td::BTreeMap> current_depth_hashes; + + auto existing_it = cells.find(root_hash); + if (existing_it != cells.end()) { + existing_it->second.should_cache = true; + } else { + current_depth_hashes.emplace(root_hash, std::make_pair(cell_count, false)); + } + int current_depth = root_depth; + int next_child_idx = cell_count + 1; + while (!current_depth_hashes.empty()) { + if (current_depth > Cell::max_depth) { + return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); + } + + cell_list.resize(cell_list.size() + current_depth_hashes.size()); + td::BTreeMap> next_depth_hashes; + auto batch_start = current_depth_hashes.begin(); + while (batch_start != current_depth_hashes.end()) { + std::vector batch_hashes; + batch_hashes.reserve(load_batch_size); + std::vector*> batch_idxs_should_cache; + batch_idxs_should_cache.reserve(load_batch_size); + + while (batch_hashes.size() < load_batch_size && batch_start != current_depth_hashes.end()) { + batch_hashes.push_back(batch_start->first.as_slice()); + batch_idxs_should_cache.push_back(&batch_start->second); + ++batch_start; + } + + TRY_RESULT_PREFIX(loaded_results, reader->load_bulk(batch_hashes), + "error while importing a cell into a bag of cells: "); + DCHECK(loaded_results.size() == batch_hashes.size()); + + for (size_t i = 0; i < loaded_results.size(); ++i) { + auto& cell = loaded_results[i]; + + if (cell->get_virtualization() != 0) { + return td::Status::Error( + "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); + } + + const auto hash = cell->get_hash(); + CellSlice cs(std::move(cell)); + + DCHECK(cs.size_refs() <= 4); + std::array refs{-1, -1, -1, -1}; + for (unsigned j = 0; j < cs.size_refs(); j++) { + auto child = cs.prefetch_ref(j); + const auto child_hash = child->get_hash(); + + auto existing_global_it = cells.find(child_hash); + if (existing_global_it != cells.end()) { + existing_global_it->second.should_cache = true; + refs[j] = existing_global_it->second.idx; + continue; + } + auto current_depth_it = current_depth_hashes.find(child_hash); + if (current_depth_it != current_depth_hashes.end()) { + current_depth_it->second.second = true; + refs[j] = current_depth_it->second.first; + continue; + } + auto next_depth_it = next_depth_hashes.find(child_hash); + if (next_depth_it != next_depth_hashes.end()) { + next_depth_it->second.second = true; + refs[j] = next_depth_it->second.first; + continue; + } + auto res = next_depth_hashes.emplace(child_hash, std::make_pair(next_child_idx, false)); + refs[j] = next_child_idx++; + } + + auto dc = cs.move_as_loaded_cell().data_cell; + auto idx_should_cache = batch_idxs_should_cache[i]; + auto res = cells.emplace(hash, CellInfo(idx_should_cache->first, std::move(refs))); + DCHECK(res.second); + cell_list[idx_should_cache->first] = &*res.first; + CellInfo& dc_info = res.first->second; + dc_info.should_cache = idx_should_cache->second; + dc_info.hcnt = static_cast(dc->get_level_mask().get_hashes_count()); + DCHECK(dc_info.hcnt <= 4); + dc_info.wt = 0; // will be calculated after traversing + TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); + data_bytes += dc_info.serialized_size = serialized_size; + cell_count++; + } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); + } + } + + current_depth_hashes = std::move(next_depth_hashes); + next_depth_hashes.clear(); + current_depth++; + } + DCHECK(next_child_idx == cell_count); + + for (int idx = cell_count - 1; idx >= start_ind; --idx) { + CellInfo& cell_info = cell_list[idx]->second; + + unsigned sum_child_wt = 1; + for (size_t j = 0; j < cell_info.ref_idx.size(); ++j) { + int child_idx = cell_info.ref_idx[j]; + if (child_idx == -1) { + continue; + } + sum_child_wt += cell_list[child_idx]->second.wt; + ++int_refs; + } + cell_info.wt = static_cast(std::min(0xffU, sum_child_wt)); } - if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); - } - auto it = cells.find(hash); - if (it != cells.end()) { - it->second.should_cache = true; - return it->second.idx; - } - TRY_RESULT(cell, reader->load_cell(hash.as_slice())); - if (cell->get_virtualization() != 0) { - return td::Status::Error( - "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); - } - CellSlice cs(std::move(cell)); - std::array refs; - std::fill(refs.begin(), refs.end(), -1); - DCHECK(cs.size_refs() <= 4); - unsigned sum_child_wt = 1; - for (unsigned i = 0; i < cs.size_refs(); i++) { - TRY_RESULT(ref, import_cell(cs.prefetch_ref(i)->get_hash(), depth + 1)); - refs[i] = ref; - sum_child_wt += cell_list[ref]->second.wt; - ++int_refs; - } - auto dc = cs.move_as_loaded_cell().data_cell; - auto res = cells.emplace(hash, CellInfo(cell_count, refs)); - DCHECK(res.second); - cell_list.push_back(&*res.first); - CellInfo& dc_info = res.first->second; - dc_info.wt = (unsigned char)std::min(0xffU, sum_child_wt); - unsigned hcnt = dc->get_level_mask().get_hashes_count(); - DCHECK(hcnt <= 4); - dc_info.hcnt = (unsigned char)hcnt; - TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); - data_bytes += dc_info.serialized_size = serialized_size; - return cell_count++; + + auto root_it = cells.find(root_hash); + DCHECK(root_it != cells.end()); + return root_it->second.idx; } void LargeBocSerializer::reorder_cells() { @@ -386,7 +466,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } DCHECK(offs == info.data_size); @@ -399,26 +479,42 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { if (logger_ptr_) { logger_ptr_->start_stage("serialize"); } - for (int i = 0; i < cell_count; ++i) { - auto hash = cell_list[cell_count - 1 - i]->first; - const auto& dc_info = cell_list[cell_count - 1 - i]->second; - TRY_RESULT(dc, reader->load_cell(hash.as_slice())); - bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; - if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { - with_hash = true; + for (int batch_start = 0; batch_start < cell_count; batch_start += load_batch_size) { + int batch_end = std::min(batch_start + static_cast(load_batch_size), cell_count); + + std::vector batch_hashes; + batch_hashes.reserve(batch_end - batch_start); + for (int i = batch_start; i < batch_end; ++i) { + int cell_index = cell_count - 1 - i; + batch_hashes.push_back(cell_list[cell_index]->first.as_slice()); } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); - writer.store_bytes(buf, s); - DCHECK(dc->size_refs() == dc_info.get_ref_num()); - unsigned ref_num = dc_info.get_ref_num(); - for (unsigned j = 0; j < ref_num; ++j) { - int k = cell_count - 1 - dc_info.ref_idx[j]; - DCHECK(k > i && k < cell_count); - store_ref(k); + + TRY_RESULT(batch_cells, reader->load_bulk(std::move(batch_hashes))); + + for (int i = batch_start; i < batch_end; ++i) { + int idx_in_batch = i - batch_start; + int cell_index = cell_count - 1 - i; + + const auto& dc_info = cell_list[cell_index]->second; + auto& dc = batch_cells[idx_in_batch]; + + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; + if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { + with_hash = true; + } + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); + DCHECK(dc->size_refs() == dc_info.get_ref_num()); + unsigned ref_num = dc_info.get_ref_num(); + for (unsigned j = 0; j < ref_num; ++j) { + int k = cell_count - 1 - dc_info.ref_idx[j]; + DCHECK(k > i && k < cell_count); + store_ref(k); + } } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); } } DCHECK(writer.position() - keep_position == info.data_size); @@ -435,7 +531,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } } // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index aab1711f4..d74ecb17f 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -63,6 +63,9 @@ bool debug(int x) { #define DEB_START DBG_START #define DEB DBG +static constexpr int randseed_idx = 6; +static constexpr int inmsgparams_idx = 17; + int exec_set_gas_generic(VmState* st, long long new_gas_limit) { if (new_gas_limit < st->gas_consumed()) { throw VmNoGas{}; @@ -152,6 +155,30 @@ int exec_get_var_param(VmState* st, unsigned idx) { return exec_get_param(st, idx, nullptr); } +int exec_get_var_param_long(VmState* st, unsigned idx) { + idx &= 255; + VM_LOG(st) << "execute GETPARAMLONG " << idx; + return exec_get_param(st, idx, nullptr); +} + +int exec_get_in_msg_param(VmState* st, unsigned idx, const char* name) { + if (name) { + VM_LOG(st) << "execute " << name; + } + Ref t = get_param(st, inmsgparams_idx).as_tuple(); + if (t.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + st->get_stack().push(tuple_index(t, idx)); + return 0; +} + +int exec_get_var_in_msg_param(VmState* st, unsigned idx) { + idx &= 15; + VM_LOG(st) << "execute INMSGPARAM " << idx; + return exec_get_in_msg_param(st, idx, nullptr); +} + int exec_get_config_dict(VmState* st) { exec_get_param(st, 9, "CONFIGDICT"); st->get_stack().push_smallint(32); @@ -358,6 +385,75 @@ int exec_get_forward_fee_simple(VmState* st) { return 0; } +int exec_get_extra_currency_balance(VmState* st) { + VM_LOG(st) << "execute GETEXTRABALANCE"; + Stack& stack = st->get_stack(); + auto id = (td::uint32)stack.pop_long_range((1LL << 32) - 1); + + auto tuple = st->get_c7(); + tuple = tuple_index(tuple, 0).as_tuple_range(255); + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + tuple = tuple_index(tuple, 7).as_tuple_range(255); // Balance + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + auto dict_root = tuple_index(tuple, 1); + if (!dict_root.is_cell() && !dict_root.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not cell or null"}; + } + + class LocalVmState : public VmStateInterface { + public: + explicit LocalVmState(VmState* st) : st_(st) { + } + ~LocalVmState() override = default; + + Ref load_library(td::ConstBitPtr hash) override { + return st_->load_library(hash); + } + void register_cell_load(const CellHash& cell_hash) override { + auto new_cell = st_->register_cell_load_free(cell_hash); + consume_gas(new_cell ? VmState::cell_load_gas_price : VmState::cell_reload_gas_price); + } + void register_cell_create() override { + // Not expected in this operation + } + int get_global_version() const override { + return st_->get_global_version(); + } + + private: + VmState* st_; + long long remaining = VmState::get_extra_balance_cheap_max_gas_price; + + void consume_gas(long long gas) { + long long consumed = std::min(gas, remaining); + st_->consume_gas(consumed); + remaining -= consumed; + if (remaining == 0) { + st_->consume_free_gas(gas - consumed); + } + } + }; + bool cheap = st->register_get_extra_balance_call(); + LocalVmState local_vm_state{st}; + VmStateInterface::Guard guard{cheap ? (VmStateInterface*)&local_vm_state : st}; + + Dictionary dict{dict_root.as_cell(), 32}; + Ref cs = dict.lookup(td::BitArray<32>(id)); + if (cs.is_null()) { + stack.push_smallint(0); + } else { + td::RefInt256 x; + util::load_var_integer_q(cs.write(), x, /* len_bits = */ 5, /* sgnd = */ false, /* quiet = */ false); + stack.push_int(std::move(x)); + } + + return 0; +} + void register_ton_config_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mkfixedrange(0xf820, 0xf823, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param)) @@ -391,11 +487,24 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf840, 16, "GETGLOBVAR", exec_get_global_var)) .insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global)) .insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var)) - .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)); + .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)) + .insert(OpcodeInstr::mksimple(0xf880, 16, "GETEXTRABALANCE", exec_get_extra_currency_balance)->require_version(10)) + .insert(OpcodeInstr::mkfixedrange(0xf88100, 0xf88111, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf88111, 24, "INMSGPARAMS", std::bind(exec_get_param, _1, 17, "INMSGPARAMS"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf88112, 0xf881ff, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf890, 16, "INMSG_BOUNCE", std::bind(exec_get_in_msg_param, _1, 0, "INMSG_BOUNCE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf891, 16, "INMSG_BOUNCED", std::bind(exec_get_in_msg_param, _1, 1, "INMSG_BOUNCED"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf892, 16, "INMSG_SRC", std::bind(exec_get_in_msg_param, _1, 2, "INMSG_SRC"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf893, 16, "INMSG_FWDFEE", std::bind(exec_get_in_msg_param, _1, 3, "INMSG_FWDFEE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf894, 16, "INMSG_LT", std::bind(exec_get_in_msg_param, _1, 4, "INMSG_LT"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf895, 16, "INMSG_UTIME", std::bind(exec_get_in_msg_param, _1, 5, "INMSG_UTIME"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf896, 16, "INMSG_ORIGVALUE", std::bind(exec_get_in_msg_param, _1, 6, "INMSG_ORIGVALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf897, 16, "INMSG_VALUE", std::bind(exec_get_in_msg_param, _1, 7, "INMSG_VALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf898, 16, "INMSG_VALUEEXTRA", std::bind(exec_get_in_msg_param, _1, 8, "INMSG_VALUEEXTRA"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf899, 16, "INMSG_STATEINIT", std::bind(exec_get_in_msg_param, _1, 9, "INMSG_STATEINIT"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf89a, 0xf8a0, 16, 4, instr::dump_1c("INMSGPARAM "), exec_get_var_in_msg_param)->require_version(11)); } -static constexpr int randseed_idx = 6; - td::RefInt256 generate_randu256(VmState* st) { auto tuple = st->get_c7(); auto t1 = tuple_index(tuple, 0).as_tuple_range(255); @@ -501,18 +610,24 @@ void register_prng_ops(OpcodeTable& cp0) { } int exec_compute_hash(VmState* st, int mode) { - VM_LOG(st) << "execute HASH" << (mode & 1 ? 'S' : 'C') << 'U'; + VM_LOG(st) << "execute HASH" << "CSB"[mode] << 'U'; Stack& stack = st->get_stack(); std::array hash; - if (!(mode & 1)) { + if (mode == 0) { // cell auto cell = stack.pop_cell(); hash = cell->get_hash().as_array(); - } else { + } else if (mode == 1) { // slice auto cs = stack.pop_cellslice(); - vm::CellBuilder cb; + CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); - // TODO: use cb.get_hash() instead - hash = cb.finalize()->get_hash().as_array(); + if (st->get_global_version() >= 12) { + hash = cb.finalize_novm()->get_hash().as_array(); + } else { + hash = cb.finalize()->get_hash().as_array(); + } + } else { // builder + auto cb = stack.pop_builder(); + hash = cb.write().finalize_novm()->get_hash().as_array(); } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); @@ -1256,6 +1371,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf916, 16, "HASHBU", std::bind(exec_compute_hash, _1, 2))->require_version(12)) .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) @@ -1380,10 +1496,13 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { return 0; } -bool skip_maybe_anycast(CellSlice& cs) { +bool skip_maybe_anycast(CellSlice& cs, int global_version) { if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; return cs.advance(1) // just$1 && cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) @@ -1391,7 +1510,7 @@ bool skip_maybe_anycast(CellSlice& cs) { && cs.advance(depth); // rewrite_pfx:(bits depth) = Anycast; } -bool skip_message_addr(CellSlice& cs) { +bool skip_message_addr(CellSlice& cs, int global_version) { switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; return true; @@ -1400,15 +1519,18 @@ bool skip_message_addr(CellSlice& cs) { return cs.fetch_uint_to(9, len) // len:(## 9) && cs.advance(len); // external_address:(bits len) = MsgAddressExt; } - case 2: { // addr_std$10 - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; + case 2: { // addr_std$10 + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } unsigned len; - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; } default: return false; @@ -1420,7 +1542,7 @@ int exec_load_message_addr(VmState* st, bool quiet) { Stack& stack = st->get_stack(); auto csr = stack.pop_cellslice(); td::Ref addr{true}; - if (util::load_msg_addr_q(csr.write(), addr.write(), quiet)) { + if (util::load_msg_addr_q(csr.write(), addr.write(), st->get_global_version(), quiet)) { stack.push_cellslice(std::move(addr)); stack.push_cellslice(std::move(csr)); if (quiet) { @@ -1433,11 +1555,14 @@ int exec_load_message_addr(VmState* st, bool quiet) { return 0; } -bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { +bool parse_maybe_anycast(CellSlice& cs, StackEntry& res, int global_version) { res = StackEntry{}; if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; Ref pfx; if (cs.advance(1) // just$1 @@ -1450,7 +1575,7 @@ bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { return false; } -bool parse_message_addr(CellSlice& cs, std::vector& res) { +bool parse_message_addr(CellSlice& cs, std::vector& res, int global_version) { res.clear(); switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; @@ -1471,9 +1596,9 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { StackEntry v; int workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_int_to(8, workchain) // workchain_id:int8 - && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_int_to(8, workchain) // workchain_id:int8 + && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; res.emplace_back(td::make_refint(2)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1483,13 +1608,16 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { break; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } StackEntry v; int len, workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.fetch_int_to(32, workchain) // workchain_id:int32 - && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.fetch_int_to(32, workchain) // workchain_id:int32 + && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; res.emplace_back(td::make_refint(3)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1508,7 +1636,7 @@ int exec_parse_message_addr(VmState* st, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector res; - if (!(parse_message_addr(cs, res) && cs.empty_ext())) { + if (!(parse_message_addr(cs, res, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); } else { @@ -1548,7 +1676,7 @@ int exec_rewrite_message_addr(VmState* st, bool allow_var_addr, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); return 0; @@ -1698,6 +1826,7 @@ int exec_send_message(VmState* st) { Ref dest; td::RefInt256 value; td::RefInt256 user_fwd_fee, user_ihr_fee; + unsigned extra_flags_len = 0; bool have_extra_currencies = false; bool ext_msg = msg.info->prefetch_ulong(1); if (ext_msg) { // External message @@ -1713,19 +1842,25 @@ int exec_send_message(VmState* st) { if (!tlb::csr_unpack(msg.info, info)) { throw VmError{Excno::unknown, "invalid message"}; } - ihr_disabled = info.ihr_disabled; + ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); - Ref extra; + Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { throw VmError{Excno::unknown, "invalid message"}; } have_extra_currencies = !extra.is_null(); user_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - user_ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (st->get_global_version() >= 12) { + user_ihr_fee = td::zero_refint(); + extra_flags_len = info.extra_flags->size(); + } else { + // Legacy: extra_flags was previously ihr_fee + user_ihr_fee = block::tlb::t_Grams.as_integer(info.extra_flags); + } } bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1); - td::Ref prices_cs; + Ref prices_cs; if (st->get_global_version() >= 6) { prices_cs = tuple_index(get_unpacked_config_tuple(st), is_masterchain ? 4 : 5).as_slice(); } else { @@ -1758,7 +1893,7 @@ int exec_send_message(VmState* st) { } else { max_cells = 1 << 13; } - vm::VmStorageStat stat(max_cells); + VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); if (st->get_global_version() >= 10 && have_extra_currencies) { @@ -1839,7 +1974,8 @@ int exec_send_message(VmState* st) { bits = 4 + my_addr->size() + dest->size() + stored_grams_len(value) + 1 + 32 + 64; td::RefInt256 fwd_fee_first = (fwd_fee * prices.first_frac) >> 16; bits += stored_grams_len(fwd_fee - fwd_fee_first); - bits += stored_grams_len(ihr_fee); + // Legacy: extra_flags was previously ihr_fee + bits += st->get_global_version() >= 12 ? extra_flags_len : stored_grams_len(ihr_fee); } // init bits++; @@ -2026,9 +2162,9 @@ bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sg bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet) { return load_var_integer_q(cs, res, 4, false, quiet); } -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet) { res = cs; - if (!skip_message_addr(cs)) { + if (!skip_message_addr(cs, global_version)) { cs = res; if (quiet) { return false; @@ -2038,10 +2174,11 @@ bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { res.cut_tail(cs); return true; } -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet) { +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet) { // Like exec_rewrite_message_addr, but for std address case std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, global_version) && cs.empty_ext())) { if (quiet) { return false; } @@ -2076,14 +2213,14 @@ td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd) { td::RefInt256 load_coins(CellSlice& cs) { return load_var_integer(cs, 4, false); } -CellSlice load_msg_addr(CellSlice& cs) { +CellSlice load_msg_addr(CellSlice& cs, int global_version) { CellSlice addr; - load_msg_addr_q(cs, addr, false); + load_msg_addr_q(cs, addr, global_version, false); return addr; } -std::pair parse_std_addr(CellSlice cs) { +std::pair parse_std_addr(CellSlice cs, int global_version) { std::pair res; - parse_std_addr_q(std::move(cs), res.first, res.second, false); + parse_std_addr_q(std::move(cs), res.first, res.second, global_version, false); return res; } diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index bbac078f2..d33505085 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -32,14 +32,15 @@ namespace util { // "_q" functions throw on error if not quiet, return false if quiet (leaving cs unchanged) bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet); bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet); -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet); -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet); +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet); +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet); // Non-"_q" functions throw on error td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd); td::RefInt256 load_coins(CellSlice& cs); -CellSlice load_msg_addr(CellSlice& cs); -std::pair parse_std_addr(CellSlice cs); +CellSlice load_msg_addr(CellSlice& cs, int global_version); +std::pair parse_std_addr(CellSlice cs, int global_version); // store_... functions throw on error if not quiet, return false if quiet (leaving cb unchanged) bool store_var_integer(CellBuilder& cb, const td::RefInt256& x, int len_bits, bool sgnd, bool quiet = false); diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 3c1118c60..356f0df22 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -656,12 +656,12 @@ bool VmState::register_library_collection(Ref lib) { } void VmState::register_cell_load(const CellHash& cell_hash) { - if (cell_load_gas_price == cell_reload_gas_price) { - consume_gas(cell_load_gas_price); - } else { - auto ok = loaded_cells.insert(cell_hash); // check whether this is the first time this cell is loaded - consume_gas(ok.second ? cell_load_gas_price : cell_reload_gas_price); - } + auto new_cell = loaded_cells.insert(cell_hash).second; // check whether this is the first time this cell is loaded + consume_gas(new_cell ? cell_load_gas_price : cell_reload_gas_price); +} + +bool VmState::register_cell_load_free(const CellHash& cell_hash) { + return loaded_cells.insert(cell_hash).second; } void VmState::register_cell_create() { @@ -708,17 +708,31 @@ Ref lookup_library_in(td::ConstBitPtr key, Ref lib_root) { void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_actions, bool return_gas, bool isolate_gas, int ret_vals) { - new_state.log = std::move(log); - new_state.libraries = std::move(libraries); + if (global_version < 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); + } new_state.stack_trace = stack_trace; new_state.max_data_depth = max_data_depth; if (!isolate_gas) { new_state.loaded_cells = std::move(loaded_cells); } else { - consume_gas(std::min(chksgn_counter, chksgn_free_count) * chksgn_gas_price); + consume_gas(free_gas_consumed); chksgn_counter = 0; + get_extra_balance_counter = 0; + free_gas_consumed = 0; + } + if (global_version >= 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); } new_state.chksgn_counter = chksgn_counter; + new_state.free_gas_consumed = free_gas_consumed; + new_state.get_extra_balance_counter = get_extra_balance_counter; + if (global_version >= 10) { + new_state.gas = GasLimits{std::min(new_state.gas.gas_limit, gas.gas_remaining), + std::min(new_state.gas.gas_max, gas.gas_remaining)}; + } auto new_parent = std::make_unique(); new_parent->return_data = return_data; @@ -743,6 +757,8 @@ void VmState::restore_parent_vm(int res) { loaded_cells = std::move(child_state.loaded_cells); } chksgn_counter = child_state.chksgn_counter; + get_extra_balance_counter = child_state.get_extra_balance_counter; + free_gas_consumed = child_state.free_gas_consumed; VM_LOG(this) << "Child VM finished. res: " << res << ", steps: " << child_state.steps << ", gas: " << child_state.gas_consumed(); @@ -769,11 +785,20 @@ void VmState::restore_parent_vm(int res) { cur_stack.push(std::move(child_state.stack->at(i))); } cur_stack.push_smallint(res); - if (parent->return_data) { - cur_stack.push_cell(child_state.get_committed_state().c4); - } - if (parent->return_actions) { - cur_stack.push_cell(child_state.get_committed_state().c5); + if (global_version >= 11 && !child_state.get_committed_state().committed) { + if (parent->return_data) { + cur_stack.push_null(); + } + if (parent->return_actions) { + cur_stack.push_null(); + } + } else { + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } } if (parent->return_gas) { cur_stack.push_smallint(child_state.gas.gas_consumed()); diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index a171ef27e..aad67d61e 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" +#include "td/utils/HashMap.h" #include "vm/cellslice.h" #include "vm/stack.hpp" #include "vm/vmstate.h" @@ -103,6 +104,8 @@ class VmState final : public VmStateInterface { td::uint16 max_data_depth = 512; // Default value int global_version{0}; size_t chksgn_counter = 0; + size_t get_extra_balance_counter = 0; + long long free_gas_consumed = 0; std::unique_ptr parent = nullptr; public: @@ -161,7 +164,10 @@ class VmState final : public VmStateInterface { bls_g2_multiexp_coef2_gas_price = 22840, bls_pairing_base_gas_price = 20000, - bls_pairing_element_gas_price = 11800 + bls_pairing_element_gas_price = 11800, + + get_extra_balance_cheap_count = 5, + get_extra_balance_cheap_max_gas_price = 200 }; VmState(); VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, @@ -214,6 +220,9 @@ class VmState final : public VmStateInterface { consume_stack_gas((unsigned)stk->depth()); } } + void consume_free_gas(long long amount) { + free_gas_consumed += amount; + } GasLimits get_gas_limits() const { return gas; } @@ -226,6 +235,7 @@ class VmState final : public VmStateInterface { Ref load_library( td::ConstBitPtr hash) override; // may throw a dictionary exception; returns nullptr if library is not found void register_cell_load(const CellHash& cell_hash) override; + bool register_cell_load_free(const CellHash& cell_hash); void register_cell_create() override; bool init_cp(int new_cp); bool set_cp(int new_cp); @@ -420,9 +430,19 @@ class VmState final : public VmStateInterface { ++chksgn_counter; if (chksgn_counter > chksgn_free_count) { consume_gas(chksgn_gas_price); + } else { + consume_free_gas(chksgn_gas_price); } } } + bool register_get_extra_balance_call() { + ++get_extra_balance_counter; + return get_extra_balance_counter <= get_extra_balance_cheap_count; + } + + td::HashSet extract_loaded_cells() { + return std::move(loaded_cells); + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/dht-server/CMakeLists.txt b/dht-server/CMakeLists.txt index 6daac0334..07c93f247 100644 --- a/dht-server/CMakeLists.txt +++ b/dht-server/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index fa9fad132..46a011352 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -147,6 +147,7 @@ ton::tl_object_ptr Config::tl() const { } std::vector> val_vec; + std::vector> col_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; @@ -169,9 +170,9 @@ ton::tl_object_ptr Config::tl() const { gc_vec->ids_.push_back(id.tl()); } return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), - ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), nullptr, + nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1194,7 +1195,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows dht-server build information", [&]() { - std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index 95ee70691..f8d49a2d8 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 77963e959..e2336e694 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -3,7 +3,9 @@ Global version is a parameter specified in `ConfigParam 8` ([block.tlb](https:// Various features are enabled depending on the global version. ## Version 4 -New features of version 4 are desctibed in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). +__Enabled in mainnet on 2023-12-20__ + +New features of version 4 are described in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). ### New TVM instructions * `PREVMCBLOCKS`, `PREVKEYBLOCK` @@ -40,6 +42,7 @@ intermediate value before division (e.g. `(xy+w)/z`). * Unpaid storage fee is now saved to `due_payment` ## Version 5 +__Enabled in mainnet on 2024-02-03__ ### Gas limits Version 5 enables higher gas limits for special contracts. @@ -57,6 +60,7 @@ See [this post](https://t.me/tonstatus/88) for details. * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. ## Version 6 +__Enabled in mainnet on 2024-03-16__ ### c7 tuple **c7** tuple extended from 14 to 17 elements: @@ -101,10 +105,12 @@ Operations for working with Merkle proofs, where cells can have non-zero level a ## Version 7 +__Enabled in mainnet on 2024-04-18__ [Explicitly nullify](https://github.com/ton-blockchain/ton/pull/957/files) `due_payment` after due reimbursment. ## Version 8 +__Enabled in mainnet on 2024-08-25__ - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. @@ -113,6 +119,7 @@ Operations for working with Merkle proofs, where cells can have non-zero level a - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. ## Version 9 +__Enabled in mainnet on 2025-02-13__ ### c7 tuple c7 tuple parameter number **13** (previous blocks info tuple) now has the third element. It contains ids of the 16 last masterchain blocks with seqno divisible by 100. @@ -137,6 +144,7 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - Fix recursive jump to continuations with non-null control data. ## Version 10 +__Enabled in mainnet on 2025-05-07__ ### Extra currencies - Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). @@ -152,7 +160,117 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - `SENDMSG` does not check the number of extra currencies. - Extra currency dictionary is not counted in the account size and does not affect storage fees. - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. +- Reserve action cannot reserve extra currencies. +Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" and "negate amount") now only affect TONs, but not extra currencies. + +### Anycast addresses and address rewrite +- Anycast addresses are not allowed in `dest` of internal and external messages. +- `addr_var` are not allowed in `dest` of external messages. + - Note: as before, `addr_var` in `dest` of internal messages are automatically replaced with `addr_std`. +- TVM instructions `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- `addr:MsgAddressInt` in `Account` cannot be an anycast address. + - Therefore, `src` of outbound messages cannot be an anycast address. + - Existing accounts with anycast addresses change to non-anycast addresses in the first transaction. +- When deploying an account with `fixed_prefix_length` in `StateInit` of the message (it was called `split_depth` before), the first `fixed_prefix_length` bits of the address are not compared against the state hash. + - This allows deploying an account to an arbitrary shard regardless of the hash of state init. + - `fixed_prefix_length` remains in the account state. + - `fixed_prefix_length` of the account can be at most 8. The limit can be changed in size limits config (`ConfigParam 43`). ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. +- New instruction `GETEXTRABALANCE` (`id - amount`). Takes id of the extra currency (integer in range `0..2^32-1`), returns the amount of this extra currency on the account balance. + - This is equivalent to taking the extra currency dictionary (`BALANCE SECOND`), loading value (`UDICTGET`) and parsing it (`LDVARUINT32`). If `id` is not present in the dictionary, `0` is returned. + - `GETEXTRABALANCE` has special gas cost that allows writing gas-efficient code with predictable gas usage even if there are a lot of different extra currencies. + - The full gas cost of `GETEXTRABALANCE` is `26` (normal instruction cost) plus gas for loading cells (up to `3300` if the dictionary has maximum depth). + - However, the first `5` executions of `GETEXTRABALANCE` cost at most `26+200` gas units. All subsequent executions cost the full price. + - `RUNVM` interacts with this instructions in the following way: + - Without "isolate gas" mode, the child VM shares `GETEXTRABALANCE` counter with the parent vm. + - With "isolate gas" mode, in the beginning of `RUNVM` the parent VM spends full gas for all already executed `GETEXTRABALANCE` and resets the counter. +- `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- Fixed bug in `RUNVM` caused by throwing out-of-gas exception with "isolate gas" enabled. +- Fixed setting gas limits in `RUNVM` after consuming "free gas" (e.g. after `CHKSIGN` instructions). + +### Other changes +- Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. + +## Version 11 +__Enabled in mainnet on 2025-07-05__ + +### c7 tuple +**c7** tuple extended from 17 to 18 elements: +* **17**: tuple with inbound message parameters. Asm opcode: `INMSGPARAMS`. + * The tuple contains: + * `bounce` (boolean) + * `bounced` (boolean) + * `src_addr` (slice) + * `fwd_fee` (int) + * `created_lt` (int) + * `created_at` (int) + * `orig_value` (int) - this is sometimes different from the value in `INCOMINGVALUE` and TVM stack because of storage fees + * `value` (int) - same as in `INCOMINGVALUE` and TVM stack. + * `value_extra` (cell or null) - same as in `INCOMINGVALUE`. + * `state_init` (cell or null) + * For external messages, tick-tock transactions and get methods: `bounce`, `bounced`, `fwd_fee`, `created_lt`, `created_at`, `orig_value`, `value` are 0, `value_extra` is null. + * For tick-tock transactions and get methods: `src_addr` is `addr_none`. + +### New TVM instructions +- `x GETPARAMLONG` - same as `x GETPARAM`, but `x` is in range `[0..254]`. Gas cost: `34`. +- `x INMSGPARAM` - equivalent to `INMSGPARAMS` `x INDEX`. Gas cost: `26`. + - Aliases: `INMSG_BOUNCE`, `INMSG_BOUNCED`, `INMSG_SRC`, `INMSG_FWDFEE`, `INMSG_LT`, `INMSG_UTIME`, `INMSG_ORIGVALUE`, `INMSG_VALUE`, `INMSG_VALUEEXTRA`, `INMSG_STATEINIT`. + +### New account storage stat +Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. + +**Storage dict** is the dictionary that stores refcnt for each cell in the account state. +This is required to help computing storage stats in the future, after collator-validator separation. + +### Other changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. + +## Version 12 + +### Extra message flags and new bounce format +Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUInteger 16)` (it's the same format). +This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. + +`(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. +If `(extra_flags & 3) = 3`, the bounced message contains the whole body of the original message. Otherwise, only the bits from the root of the original body are returned. +When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): +``` +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; + +new_bounce_body#fffffffe + original_body:^Cell + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; +``` +- `original_body` - cell that contains the body of the original message. If `extra_flags & 2` then the whole body is returned, otherwise it is only the root without refs. +- `original_info` - value, lt and unixtime of the original message. +- `bounced_by_phase`: + - `0` - compute phase was skipped. `exit_code` denotes the skip reason: + - `exit_code = -1` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = -2` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = -3` - no gas. + - `exit_code = -4` - account is suspended. + - `1` - compute phase failed. `exit_code` is the value from the compute phase. + - `2` - action phase failed. `exit_code` is the value from the action phase. +- `exit_code` - 32-bit exit code, see above. +- `compute_phase` - exists if it was not skipped (`bounced_by_phase > 0`): + - `gas_used`, `vm_steps` - same as in `TrComputePhase` of the transaction. + +### New TVM instructions +- `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. +- `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. + +### Other TVM changes +- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. +- `SENDMSG` instruction treats `extra_flags` field accordingly (see above). + +### Other changes +- Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). + - The previous limit was the same as in basechain (`65536`). \ No newline at end of file diff --git a/doc/TestGrams-HOWTO b/doc/TestGrams-HOWTO deleted file mode 100644 index 8156e9757..000000000 --- a/doc/TestGrams-HOWTO +++ /dev/null @@ -1,163 +0,0 @@ -The aim of this text is to describe how to quickly obtain a small amount of test Grams for test purposes, or a larger amount of test Grams for running a validator in the test network. We assume familiarity with the TON Blockchain LiteClient as explained in the LiteClient-HOWTO, and with the procedure required to compile the LiteClient and other software. For obtaining larger amount of test Grams required for running a validator, we also assume acquaintance with the FullNode-HOWTO and Validator-HOWTO. You will also need a dedicated server powerful enough for running a Full Node in order to obtain the larger amount of test Grams. Obtaining small amounts of test Grams does not require a dedicated server and may be done in several minutes on a home computer. - -1. Proof-of-Work TestGiver smart contracts -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to prevent a small number of malicious parties to collect all test Grams reserved for test purposes, a special kind of "Proof-of-Work TestGiver" smart contracts have been deployed in the masterchain of the test network. The addresses of these smart contacts are: - -Small testgivers (deliver from 10 to 100 test Grams every several minutes): - -kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN -kf8SYc83pm5JkGt0p3TQRkuiM58O9Cr3waUtR9OoFq716lN- -kf-FV4QTxLl-7Ct3E6MqOtMt-RGXMxi27g4I645lw6MTWraV -kf_NSzfDJI1A3rOM0GQm7xsoUXHTgmdhN5-OrGD8uwL2JMvQ -kf8gf1PQy4u2kURl-Gz4LbS29eaN4sVdrVQkPO-JL80VhOe6 -kf8kO6K6Qh6YM4ddjRYYlvVAK7IgyW8Zet-4ZvNrVsmQ4EOF -kf-P_TOdwcCh0AXHhBpICDMxStxHenWdLCDLNH5QcNpwMHJ8 -kf91o4NNTryJ-Cw3sDGt9OTiafmETdVFUMvylQdFPoOxIsLm -kf9iWhwk9GwAXjtwKG-vN7rmXT3hLIT23RBY6KhVaynRrIK7 -kf8JfFUEJhhpRW80_jqD7zzQteH6EBHOzxiOhygRhBdt4z2N - -Large testgivers (deliver 10000 test Grams at least once a day): - -kf8guqdIbY6kpMykR8WFeVGbZcP2iuBagXfnQuq0rGrxgE04 -kf9CxReRyaGj0vpSH0gRZkOAitm_yDHvgiMGtmvG-ZTirrMC -kf-WXA4CX4lqyVlN4qItlQSWPFIy00NvO2BAydgC4CTeIUme -kf8yF4oXfIj7BZgkqXM6VsmDEgCqWVSKECO1pC0LXWl399Vx -kf9nNY69S3_heBBSUtpHRhIzjjqY0ChugeqbWcQGtGj-gQxO -kf_wUXx-l1Ehw0kfQRgFtWKO07B6WhSqcUQZNyh4Jmj8R4zL -kf_6keW5RniwNQYeq3DNWGcohKOwI85p-V2MsPk4v23tyO3I -kf_NSPpF4ZQ7mrPylwk-8XQQ1qFD5evLnx5_oZVNywzOjSfh -kf-uNWj4JmTJefr7IfjBSYQhFbd3JqtQ6cxuNIsJqDQ8SiEA -kf8mO4l6ZB_eaMn1OqjLRrrkiBcSt7kYTvJC_dzJLdpEDKxn - -The first ten smart contracts enable a tester willing to obtain a small amount of test Grams to obtain some without spending too much computing power (typically, several minutes of work of a home computer should suffice). The remaining smart contracts are for obtaining larger amounts of test Grams required for running a validator in the test network; typically, a day of work of a dedicated server powerful enough to run a validator should suffice to obtain the necessary amount. - -You should randomly choose one of these "proof-of-work testgiver" smart contracts (from one of these two lists depending on your purpose) and obtain test Grams from this smart contract by a procedure similar to "mining". Essentially, you have to present an external message containing the proof of work and the address of your wallet to the chosen "proof-of-work testgiver" smart contract, and then the necessary amount will be sent to you. - -2. The "mining" process -~~~~~~~~~~~~~~~~~~~~~~~ - -In order to create an external message containing the "proof of work", you should run a special "mining" utility, compiled from the TON sources located in the GitHub repository. The utility is located in file './crypto/pow-miner' with respect to the build directory, and can be compiled by typing 'make pow-miner' in the build directory. - -However, before running "pow-miner", you need to know the actual values of "seed" and "complexity" parameters of the chosen "proof-of-work testgiver" smart contract. This can be done by invoking get-method "get_pow_params" of this smart contract. For instance, if you use testgiver smart contract kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN, you can simply type - - > runmethod kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN get_pow_params - -in the LiteClient console and obtain output like - - ... - arguments: [ 101616 ] - result: [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - remote result (not to be trusted): [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - -The two first large numbers in the "result:" line are the "seed" and the "complexity" of this smart contract. In this example, the seed is 229760179690128740373110445116482216837 and the complexity is 53919893334301279589334030174039261347274288845081144962207220498432. - -Next, you invoke the pow-miner utility as follows: - - $ crypto/pow-miner -vv -w -t - -Here is the number of CPU cores that you want to use for mining, is the maximal amount of seconds that the miner would run before admitting failure, is the address of your wallet (possibly not initialized yet), either in the masterchain or in the workchain (note that you need a masterchain wallet to control a validator), and are the most recent values obtained by running get-method 'get-pow-params', is the address of the chosen proof-of-work testgiver smartcontract, and is the filename of the output file where the external message with the proof of work will be saved in the case of success. - -For example, if your wallet address is kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7, you might run - - $ crypto/pow-miner -vv -w7 -t100 kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN mined.boc - -The program will run for some time (at most 100 seconds in this case) and either terminate successfully (with zero exit code) and save the required proof of work into file "mined.boc", or terminate with a non-zero exit code if no proof of work was found. - -In the case of failure, you will see something like - - [ expected required hashes for success: 2147483648 ] - [ hashes computed: 1192230912 ] - -and the program will terminate with a non-zero exit code. Then you have to obtain the "seed" and "complexity" again (because they may have changed in the meantime as a result of processing requests from more successful miners) and re-run the "pow-miner" with the new parameters, repeating the process again and again until success. - -In the case of success, you will see something like - - [ expected required hashes for success: 2147483648 ] - 4D696E65005EFE49705690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4ACDA33755876665780BAE9BE8A4D6385A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4 - Saving 176 bytes of serialized external message into file `mined.boc` - [ hashes computed: 1122036095 ] - -Then you can use the LiteClient to send external message from file "mined.boc" to the proof-of-work testgiver smart contract (and you must do this as soon as possible): - -> sendfile mined.boc -... external message status is 1 - -You can wait for several seconds and check the state of your wallet: - -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:1) - bits:(var_uint len:1 value:111) - public_cells:(var_uint len:0 value:0)) last_paid:1593722498 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720869000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:100000000000)) - other:(extra_currencies - dict:hme_empty)) - state:account_uninit)) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F12025BC2F7F2341000001C169E9DCD0945D21DBA0004_} -last transaction lt = 7720869000001 hash = 83C15CDED025970FEF7521206E82D2396B462AADB962C7E1F4283D88A0FAB7D4 -account balance is 100000000000ng - -If nobody has sent a valid proof of work with this *seed* and *complexity* before you, the proof-of-work testgiver will accept your proof of work and this will be reflected in the balance of your wallet (10 or 20 seconds may elapse after sending the external message before this happens; be sure to make several attempts and type "last" each time before checking the balance of your wallet to refresh the LiteClient state). In the case of success, you will see that the balance has been increased (and even that your wallet has been created in uninitialized state if it did not exist before). In the case of failure, you will have to obtain the new "seed" and "complexity" and repeat the mining process from the very beginning. - -If you have been lucky and the balance of your wallet has been increased, you may want to initialize the wallet if it wasn't initialized before (more information on wallet creation can be found in LiteClient-HOWTO): - -> sendfile new-wallet-query.boc -... external message status is 1 -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:3) - bits:(var_uint len:2 value:1147) - public_cells:(var_uint len:0 value:0)) last_paid:1593722691 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720945000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:99995640998)) - other:(extra_currencies - dict:hme_empty)) - state:(account_active - ( - split_depth:nothing - special:nothing - code:(just - value:(raw@^Cell - x{} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - )) - data:(just - value:(raw@^Cell - x{} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} - )) - library:hme_empty)))) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1206811EC2F7F23A1800001C16B0BC790945D20D1929934_} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} -last transaction lt = 7720945000001 hash = 73353151859661AB0202EA5D92FF409747F201D10F1E52BD0CBB93E1201676BF -account balance is 99995640998ng - -Now you are a happy owner of 100 test Grams that can be used for whatever testing purposes you had in mind. Congratulations! - -3. Automating the mining process in the case of failure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you fail to obtain your test Grams for a long time, this may happen because too many other testers are simultaneously "mining" from the same proof-of-work testgiver smart contract. Maybe you should choose another proof-of-work testgiver smart contract from one of the lists given above. Alternatively, you can write a simple script to automatically run `pow-miner` with the correct parameters again and again until success (detected by checking the exit code of `pow-miner`) and invoke the lite-client with parameter -c 'sendfile mined.boc' to send the external message immediately after it is found. - diff --git a/doc/Tests.md b/doc/Tests.md index c883731a9..4ae21a75a 100644 --- a/doc/Tests.md +++ b/doc/Tests.md @@ -3,13 +3,13 @@ TON contains multiple unit-tests, that facilitate detection of erroneous blockch ## Build tests Go inside the build directory and, if you use ninja, build the tests using the following command: -```ninja test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```ninja test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` For more details on how to build TON artifacts, please refer to any of Github actions. For cmake use: -```cmake --build . --target test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```cmake --build . --target test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` ## Run tests Go inside the build directory and with ninja execute: diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 663c8fd26..63cb28d6a 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(EMULATOR_STATIC "Build emulator as static library" OFF) set(EMULATOR_STATIC_SOURCE @@ -32,6 +30,17 @@ else() target_link_libraries(emulator PUBLIC emulator_static git) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) if (EMULATOR_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index efb14eff9..7b8e9dd1e 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -11,6 +11,7 @@ struct TransactionEmulationParams { uint32_t utime; uint64_t lt; td::optional rand_seed_hex; + td::optional prev_blocks_info; bool ignore_chksig; bool is_tick_tock; bool is_tock; @@ -49,6 +50,11 @@ td::Result decode_transaction_emulation_params(const TRY_RESULT(is_tock, td::get_json_object_bool_field(obj, "is_tock", true, false)); params.is_tock = is_tock; + TRY_RESULT(prev_blocks_info_str, td::get_json_object_string_field(obj, "prev_blocks_info", true)); + if (prev_blocks_info_str.size() > 0) { + params.prev_blocks_info = prev_blocks_info_str; + } + if (is_tock && !is_tick_tock) { return td::Status::Error("Inconsistent parameters is_tick_tock=false, is_tock=true"); } @@ -200,12 +206,18 @@ const char *emulate_with_emulator(void* em, const char* libs, const char* accoun rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); } + bool prev_blocks_set = true; + if (decoded_params.prev_blocks_info) { + prev_blocks_set = transaction_emulator_set_prev_blocks_info(em, decoded_params.prev_blocks_info.unwrap().c_str()); + } + if (!transaction_emulator_set_libs(em, libs) || !transaction_emulator_set_lt(em, decoded_params.lt) || !transaction_emulator_set_unixtime(em, decoded_params.utime) || !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || - !rand_seed_set) { + !rand_seed_set || + !prev_blocks_set) { transaction_emulator_destroy(em); return strdup(R"({"fail":true,"message":"Can't set params"})"); } diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index eb5ff9f9e..2addd353c 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -187,7 +187,7 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, external_not_accepted->elapsed_time); } - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -260,7 +260,8 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction } auto emulation_result = result.move_as_ok(); - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -638,10 +639,16 @@ const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const return strdup(jb.string_builder().as_cslice().c_str()); } -const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { +struct TvmEulatorEmulateRunMethodResponse +{ + const char *response; + const char *log; +}; + +TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { auto params_cell = vm::std_boc_deserialize(td::Slice(params_boc, len)); if (params_cell.is_error()) { - return nullptr; + return { nullptr, nullptr }; } auto params_cs = vm::load_cell_slice(params_cell.move_as_ok()); auto code = params_cs.fetch_ref(); @@ -656,12 +663,12 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc td::Ref stack; if (!vm::Stack::deserialize_to(stack_cs, stack)) { - return nullptr; + return { nullptr, nullptr }; } td::Ref c7; if (!vm::Stack::deserialize_to(c7_cs, c7)) { - return nullptr; + return { nullptr, nullptr }; } auto emulator = new emulator::TvmEmulator(code, data); @@ -676,7 +683,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc vm::CellBuilder stack_cb; if (!result.stack->serialize(stack_cb)) { - return nullptr; + return { nullptr, nullptr }; } vm::CellBuilder cb; @@ -686,7 +693,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc auto ser = vm::std_boc_serialize(cb.finalize()); if (!ser.is_ok()) { - return nullptr; + return { nullptr, nullptr }; } auto sok = ser.move_as_ok(); @@ -695,7 +702,24 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc memcpy(rn, &sz, 4); memcpy(rn+4, sok.data(), sz); - return rn; + return { rn, strdup(result.vm_log.data()) }; +} + +const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return result.response; +} + +void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return new TvmEulatorEmulateRunMethodResponse(result); +} + +void run_method_detailed_result_destroy(void *detailed_result) { + auto result = static_cast(detailed_result); + free(const_cast(result->response)); + free(const_cast(result->log)); + delete result; } const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *message_body_boc) { @@ -772,6 +796,12 @@ void emulator_config_destroy(void *config) { delete static_cast(config); } +void string_destroy(const char *str) { + if (str != nullptr) { + free(const_cast(str)); + } +} + const char* emulator_version() { auto version_json = td::JsonBuilder(); auto obj = version_json.enter_object(); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index 14879e1ec..2da767456 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -254,6 +254,27 @@ EMULATOR_EXPORT const char *tvm_emulator_run_get_method(void *tvm_emulator, int */ EMULATOR_EXPORT const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit); +/** + * @brief Optimized version of "run get method" with all passed parameters in a single call. Also returns log. + * @param len Length of params_boc buffer + * @param params_boc BoC serialized parameters, scheme: request$_ code:^Cell data:^Cell stack:^VmStack params:^[c7:^VmStack libs:^Cell] method_id:(## 32) + * @param gas_limit Gas limit + * @return Pointer to struct with two fields: + * - response: Char* with first 4 bytes defining length, and the rest BoC serialized result + * Scheme: result$_ exit_code:(## 32) gas_used:(## 32) stack:^VmStack + * - log: Char* with VM log string + */ +EMULATOR_EXPORT void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit); + +/** + * @brief Destroy detailed result of "tvm_emulator_emulate_run_method_detailed" + * @param detailed_result Pointer to detailed result struct returned by "tvm_emulator_emulate_run_method_detailed" + * + * Caller should not use string_destroy() for fields of this struct, + * as they are already freed in this function. + */ +EMULATOR_EXPORT void run_method_detailed_result_destroy(void *detailed_result); + /** * @brief Send external message * @param tvm_emulator Pointer to TVM emulator @@ -315,6 +336,15 @@ EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator); */ EMULATOR_EXPORT void emulator_config_destroy(void *config); +/** + * @brief Destroy string created by emulator library + * @param string Pointer to string to destroy + * + * This function should be used to free strings returned by emulator library functions. + * It is not safe to use caller's free() on them, as they may have been allocated using a different allocator. + */ +EMULATOR_EXPORT void string_destroy(const char *string); + /** * @brief Get git commit hash and date of the library */ diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index bd991cd73..2cec6fed9 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -27,4 +27,7 @@ _tvm_emulator_send_external_message _tvm_emulator_send_internal_message _tvm_emulator_destroy _tvm_emulator_emulate_run_method +_tvm_emulator_emulate_run_method_detailed +_run_method_detailed_result_destroy +_string_destroy _emulator_version diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index ae273ddfd..4057f5816 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -221,7 +221,7 @@ TEST(Emulator, wallet_int_and_ext_msg) { { vm::CellBuilder cb; block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); - msg_info.ihr_fee = cb.as_cellslice_ref(); + msg_info.extra_flags = cb.as_cellslice_ref(); } msg_info.created_lt = 0; msg_info.created_at = static_cast(utime); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 6267f9bd0..abdf0d2cf 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -141,7 +141,7 @@ td::Result TransactionEmulator::emulate_t return td::Status::Error("account hash mismatch"); } - return emulation_result; + return std::move(emulation_result); } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { return td::Status::Error( PSTRING() diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index eae109f40..848548c32 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -33,14 +33,18 @@ class TransactionEmulator { virtual ~EmulationResult() = default; }; - struct EmulationSuccess: EmulationResult { + struct EmulationSuccess : EmulationResult { td::Ref transaction; block::Account account; td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) - {} + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, + td::Ref actions_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_) + , transaction(transaction_) + , account(std::move(account_)) + , actions(actions_) { + } }; struct EmulationExternalNotAccepted: EmulationResult { diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index 0101ab641..b0f998a8b 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -3,7 +3,7 @@ # Sets the minimum version of CMake required to build the native library. -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index 2a3056071..8549dc0e5 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/git_watcher.cmake b/git_watcher.cmake index 78e57ba14..5993916c4 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -315,4 +315,14 @@ function(Main) endfunction() # And off we go... -Main() +option(ENABLE_GIT_WATCHER "Use git tree information in the version strings" ON) + +if (ENABLE_GIT_WATCHER) + Main() +else() + add_custom_target(check_git) + set(GIT_RETRIEVED_STATE false) + set(GIT_IS_DIRTY false) + set(GIT_COMMIT_BODY "\"\"") + configure_file("${PRE_CONFIGURE_FILE}" "${POST_CONFIGURE_FILE}" @ONLY) +endif() diff --git a/http/CMakeLists.txt b/http/CMakeLists.txt index 4a3fccf82..891525446 100644 --- a/http/CMakeLists.txt +++ b/http/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(HTTP_SOURCE http.h http.cpp diff --git a/keyring/CMakeLists.txt b/keyring/CMakeLists.txt index f8f610f2f..6a5599e98 100644 --- a/keyring/CMakeLists.txt +++ b/keyring/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYRING_SOURCE keyring.h keyring.hpp diff --git a/keys/CMakeLists.txt b/keys/CMakeLists.txt index e80436b7b..1b50236dd 100644 --- a/keys/CMakeLists.txt +++ b/keys/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYS_SOURCE keys.cpp encryptor.cpp diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b28a14e9a..03d050095 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h query-utils.hpp query-utils.cpp) target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto) diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index a0e48e64a..d6b798222 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -99,12 +99,12 @@ class ExtClientImpl : public ExtClient { promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_status, server_idx, false); } promise.set_result(std::move(R)); }; LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -163,18 +163,19 @@ class ExtClientImpl : public ExtClient { explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, true); } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, false); } private: td::actor::ActorId parent_; size_t idx_; }; - LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); - server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.hostname << ") for query " + << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), server_idx)); } @@ -199,19 +200,31 @@ class ExtClientImpl : public ExtClient { return; } for (Server& server : servers_) { - if (server.timeout && server.timeout.is_in_past()) { - LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + if (!server.timeout) { + continue; + } + if (server.timeout.is_in_past()) { + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.hostname << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; + server.timeout = {}; + } else { + alarm_timestamp().relax(server.timeout); } } } - void on_server_error(size_t idx) { - servers_[idx].alive = false; - servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + void on_server_status(size_t idx, bool ok) { + if (ok) { + if (connect_to_all_) { + servers_[idx].alive = true; + servers_[idx].ignore_until = td::Timestamp::never(); + } + } else { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + } } }; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1050e6d27..9e260dea0 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -108,7 +108,7 @@ void TestNode::run() { if (single_liteserver_idx_ != -1) { // Use single liteserver from config CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " - << servers[single_liteserver_idx_].addr << "\n"; + << servers[single_liteserver_idx_].hostname << "\n"; servers = {servers[single_liteserver_idx_]}; } } @@ -422,7 +422,12 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt } td::TerminalIO::out() << "latest masterchain block known to server is " << blkid.to_str(); if (created > 0) { - td::TerminalIO::out() << " created at " << created << " (" << now() - created << " seconds ago)\n"; + auto time = now(); + if (time >= created) { + td::TerminalIO::out() << " created at " << created << " (" << time - created << " seconds ago)\n"; + } else { + td::TerminalIO::out() << " created at " << created << " (" << created - time << " seconds in the future)\n"; + } } else { td::TerminalIO::out() << "\n"; } @@ -1627,27 +1632,93 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add } bool TestNode::get_msg_queue_sizes() { - auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); - return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); + ton::BlockIdExt blkid = mc_last_id_; + if (!blkid.is_valid_full()) { + return set_error("must obtain last block information before making other queries"); + } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } + auto b = + ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blkid)); + LOG(INFO) << "requesting recent shard configuration"; + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result R) -> void { + if (R.is_error()) { return; } - auto F = ton::fetch_tl_object(res.move_as_ok(), true); + auto F = ton::fetch_tl_object(R.move_as_ok(), true); if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getOutMsgQueueSizes"; - return; + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::get_msg_queue_sizes_cont, blkid, std::move(f->data_)); } - td::actor::send_closure_later(Self, &TestNode::got_msg_queue_sizes, F.move_as_ok()); }); } -void TestNode::got_msg_queue_sizes(ton::tl_object_ptr f) { +void TestNode::get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data) { + LOG(INFO) << "got shard configuration with respect to block " << mc_blkid.to_str(); + std::vector blocks; + blocks.push_back(mc_blkid); + auto R = vm::std_boc_deserialize(data.clone()); + if (R.is_error()) { + set_error(R.move_as_error_prefix("cannot deserialize shard configuration: ")); + return; + } + auto root = R.move_as_ok(); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { + set_error("cannot extract shard block list from shard configuration"); + return; + } + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + blocks.push_back(ref->top_block_id()); + } + } + + struct QueryInfo { + std::vector blocks; + std::vector sizes; + size_t pending; + }; + auto info = std::make_shared(); + info->blocks = std::move(blocks); + info->sizes.resize(info->blocks.size(), 0); + info->pending = info->blocks.size(); + + for (size_t i = 0; i < info->blocks.size(); ++i) { + ton::BlockIdExt block_id = info->blocks[i]; + auto b = ton::create_serialize_tl_object( + 0, ton::create_tl_lite_block_id(block_id), false); + LOG(DEBUG) << "requesting queue size for block " << block_id.to_str(); + envelope_send_query(std::move(b), [=, this](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + set_error(F.move_as_error_prefix("failed to get queue size: ")); + return; + } + auto f = F.move_as_ok(); + LOG(DEBUG) << "got queue size for block " << block_id.to_str() << " : " << f->size_; + info->sizes[i] = f->size_; + if (--info->pending == 0) { + get_msg_queue_sizes_finish(std::move(info->blocks), std::move(info->sizes)); + } + }); + } +} + +void TestNode::get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes) { + CHECK(blocks.size() == sizes.size()); td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; - for (auto &x : f->shards_) { - td::TerminalIO::out() << ton::create_block_id(x->id_).id.to_str() << " " << x->size_ << std::endl; + for (size_t i = 0; i < blocks.size(); ++i) { + td::TerminalIO::out() << blocks[i].id.to_str() << " " << sizes[i] << std::endl; } - td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ << std::endl; } bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d..4adb80380 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -143,7 +143,7 @@ class TestNode : public td::actor::Actor { ton::LogicalTime end_lt{0}; ton::Bits256 vset_hash; Ref vset_root; - std::unique_ptr vset; + std::shared_ptr vset; std::map vset_map; int special_idx{-1}; std::pair created_total, created_special; @@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor { void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, unsigned refs, td::Bits256 chash, std::string filename); bool get_msg_queue_sizes(); - void got_msg_queue_sizes(ton::tl_object_ptr f); + void get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data); + void get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes); bool get_dispatch_queue_info(ton::BlockIdExt block_id); bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr); void got_dispatch_queue_info(ton::BlockIdExt block_id, diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index b46d46a5c..7c9e7cfc7 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -318,16 +318,25 @@ bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { td::Result> LiteServerConfig::parse_global_config( const ton_api::liteclient_config_global& config) { std::vector servers; + auto get_hostname = [](const auto& f) -> std::string { + if (f->hostname_.empty()) { + return PSTRING() << td::IPAddress::ipv4_to_str(f->ip_) << ":" << f->port_; + } + if (f->port_ == 0) { + return f->hostname_; + } + return PSTRING() << f->hostname_ << ":" << f->port_; + }; for (const auto& f : config.liteservers_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = true; servers.push_back(std::move(server)); } for (const auto& f : config.liteservers_v2_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = false; for (const auto& slice_obj : f->slices_) { diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp index 28500e266..d21424695 100644 --- a/lite-client/query-utils.hpp +++ b/lite-client/query-utils.hpp @@ -73,11 +73,14 @@ struct LiteServerConfig { public: ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress addr; + std::string hostname; LiteServerConfig() = default; - LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) - : is_full(true), adnl_id(adnl_id), addr(addr) { + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, std::string hostname) + : is_full(true), adnl_id(adnl_id), hostname(std::move(hostname)) { + } + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress ip) + : is_full(true), adnl_id(adnl_id), hostname(PSTRING() << ip.get_ip_str() << ":" << ip.get_port()) { } bool accepts_query(const QueryInfo& query_info) const; diff --git a/memprof/CMakeLists.txt b/memprof/CMakeLists.txt index 2ccf11dfd..677f30511 100644 --- a/memprof/CMakeLists.txt +++ b/memprof/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(MEMPROF_SOURCE memprof/memprof.cpp memprof/memprof.h diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index ab9722a60..c2c9dc617 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 7def4a2d3..52cc40cf6 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -196,7 +196,6 @@ void OverlayImpl::add_peer(OverlayNode node) { if (R.is_error()) { VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " << R.move_as_error(); - UNREACHABLE(); return; } } @@ -247,12 +246,16 @@ void OverlayImpl::add_peers(const tl_object_ptr &nodes } } -void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (overlay_type_ == OverlayType::FixedMemberList) { +void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time) { + if (overlay_type_ == OverlayType::FixedMemberList && (!success || store_ping_time < 0.0)) { return; } if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); + if (store_ping_time >= 0.0 && success) { + p->last_ping_at = td::Timestamp::now(); + p->last_ping_time = store_ping_time; + } if (p->is_alive()) { peer_list_.bad_peers_.erase(peer); } else { @@ -261,9 +264,9 @@ void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { } } -void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); return; @@ -278,9 +281,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); return; @@ -318,9 +321,9 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -367,9 +370,9 @@ void OverlayImpl::send_random_peers_v2_cont(adnl::AdnlNodeIdShort src, OverlayNo auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -392,6 +395,26 @@ void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise R) { + if (R.is_error()) { + VLOG(OVERLAY_INFO) << oid << " ping to " << peer << " failed : " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::receive_pong, peer, timer.elapsed()); + }); + td::actor::send_closure(manager_, &OverlayManager::send_query, peer, local_id_, overlay_id_, "overlay ping", + std::move(P), td::Timestamp::in(5.0), create_serialize_tl_object()); + } +} + +void OverlayImpl::receive_pong(adnl::AdnlNodeIdShort peer, double elapsed) { + on_ping_result(peer, true, elapsed); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { if (peer_list_.peers_.size() == 0) { return; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 30a40b1cf..cad18bedc 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -134,6 +134,11 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, + td::Promise promise) { + promise.set_value(create_serialize_tl_object()); +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -319,7 +324,7 @@ void OverlayImpl::alarm() { } } } else { - VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate ist invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); } if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { @@ -356,8 +361,19 @@ void OverlayImpl::alarm() { } alarm_timestamp() = td::Timestamp::in(1.0); } else { - update_neighbours(0); - alarm_timestamp() = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + if (update_neighbours_at_.is_in_past()) { + update_neighbours(0); + update_neighbours_at_ = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + } + if (opts_.private_ping_peers_) { + if (private_ping_peers_at_.is_in_past()) { + ping_random_peers(); + private_ping_peers_at_ = td::Timestamp::in(td::Random::fast(30.0, 50.0)); + } + alarm_timestamp().relax(private_ping_peers_at_); + } + alarm_timestamp().relax(update_neighbours_at_); + alarm_timestamp().relax(update_throughput_at_); } } @@ -483,7 +499,7 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { if (!has_valid_membership_certificate()) { - VLOG(OVERLAY_WARNING) << "meber certificate is invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); return; } @@ -721,6 +737,9 @@ void OverlayImpl::get_stats(td::Promiseis_alive_ = peer.is_alive(); node_obj->node_flags_ = peer.get_node()->flags(); + node_obj->last_ping_at_ = (peer.last_ping_at ? peer.last_ping_at.at_unix() : -1.0); + node_obj->last_ping_time_ = peer.last_ping_time; + res->nodes_.push_back(std::move(node_obj)); }); diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 41a04dec2..7727f5f1f 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -105,11 +105,11 @@ class OverlayPeer { void on_ping_result(bool success) { if (success) { missed_pings_ = 0; - last_ping_at_ = td::Timestamp::now(); + last_receive_at_ = td::Timestamp::now(); is_alive_ = true; } else { ++missed_pings_; - if (missed_pings_ >= 3 && last_ping_at_.is_in_past(td::Timestamp::in(-15.0))) { + if (missed_pings_ >= 3 && last_receive_at_.is_in_past(td::Timestamp::in(-15.0))) { is_alive_ = false; } } @@ -149,6 +149,9 @@ class OverlayPeer { td::string ip_addr_str = "undefined"; + td::Timestamp last_ping_at = td::Timestamp::never(); + double last_ping_time = -1.0; + private: OverlayNode node_; adnl::AdnlNodeIdShort id_; @@ -157,7 +160,7 @@ class OverlayPeer { size_t missed_pings_ = 0; bool is_alive_ = true; bool is_permanent_member_ = false; - td::Timestamp last_ping_at_ = td::Timestamp::now(); + td::Timestamp last_receive_at_ = td::Timestamp::now(); }; class OverlayImpl : public Overlay { @@ -195,13 +198,15 @@ class OverlayImpl : public Overlay { alarm_timestamp() = td::Timestamp::in(1); } - void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); - void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); - void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); + void on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time = -1.0); + void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void ping_random_peers(); + void receive_pong(adnl::AdnlNodeIdShort peer, double elapsed); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { @@ -335,6 +340,7 @@ class OverlayImpl : public Overlay { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -390,9 +396,10 @@ class OverlayImpl : public Overlay { td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; - td::Timestamp update_throughput_at_; - td::Timestamp update_neighbours_at_; + td::Timestamp update_throughput_at_ = td::Timestamp::now(); + td::Timestamp update_neighbours_at_ = td::Timestamp::now(); td::Timestamp last_throughput_update_; + td::Timestamp private_ping_peers_at_ = td::Timestamp::now(); std::unique_ptr callback_; diff --git a/overlay/overlays.h b/overlay/overlays.h index 5eb63b13f..972e48cce 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -270,6 +270,7 @@ struct OverlayOptions { td::uint32 propagate_broadcast_to_ = 5; td::uint32 default_permanent_members_flags_ = 0; double broadcast_speed_multiplier_ = 1.0; + bool private_ping_peers_ = false; }; class Overlays : public td::actor::Actor { diff --git a/recent_changelog.md b/recent_changelog.md index 820d2aa42..6b65d2183 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,3 @@ -## 2025.03 Update -1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) -2. Optmization of validation process, in particular CellStorageStat. -3. Flag for speeding up broadcasts in various overlays. -4. Fixes for static builds for emulator and tonlibjson -5. Improving getstats output: add - * Liteserver queries count - * Collated/validated blocks count, number of active sessions - * Persistent state sizes - * Initial sync progress -6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib +## 2025.07 Accelerator Update -Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index f7e30c802..f7a9d73d9 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp rldp2 dht tonlib git) diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 39e0d3ca8..688476328 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index c144ec01d..984e7bb1a 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 30403f5e2..5a53e7398 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/MicrochunkTree.cpp b/storage/MicrochunkTree.cpp index 3249d25b4..1c7359336 100644 --- a/storage/MicrochunkTree.cpp +++ b/storage/MicrochunkTree.cpp @@ -188,7 +188,8 @@ td::Result> MicrochunkTree::get_proof(td::uint64 l, td::uint64 if (!torrent.inited_info()) { return td::Status::Error("Torrent info is not ready"); } - if (!torrent.get_info().piece_size % MICROCHUNK_SIZE != 0) { + // piece_size must be an exact multiple of MICROCHUNK_SIZE + if ((torrent.get_info().piece_size % MICROCHUNK_SIZE) != 0) { return td::Status::Error("Invalid piece size in torrent"); } td::Ref root_raw = vm::CellSlice(vm::NoVm(), root_proof_).prefetch_ref(); diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index e24fffff5..92a83968f 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -27,6 +27,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" +#include "common/delay.h" #include "td/actor/MultiPromise.h" namespace ton { @@ -545,8 +546,9 @@ void NodeActor::loop_queries() { auto it = peers_.find(part.peer_id); CHECK(it != peers_.end()); auto &state = it->second.state; - CHECK(state->peer_state_ready_); - CHECK(state->peer_state_.load().will_upload); + if (!state->peer_state_ready_ || !state->peer_state_.load().will_upload) { + continue; + } CHECK(state->node_queries_active_.size() < MAX_PEER_TOTAL_QUERIES); auto part_id = part.part_id; if (state->node_queries_active_.insert(static_cast(part_id)).second) { @@ -787,15 +789,21 @@ void NodeActor::db_store_torrent_meta() { return; } next_db_store_meta_at_ = td::Timestamp::never(); - auto meta = torrent_.get_meta_str(); - db_->set(create_hash_tl_object(torrent_.get_hash()), td::BufferSlice(meta), - [new_count = (td::int64)torrent_.get_ready_parts_count(), SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); - } - }); + auto meta = torrent_.get_meta(); + delay_action( + [SelfId = actor_id(this), meta = std::move(meta), db = db_, hash = torrent_.get_hash(), + new_count = (td::int64)torrent_.get_ready_parts_count()]() { + auto meta_str = meta.serialize(); + db->set(create_hash_tl_object(hash), td::BufferSlice(meta_str), + [=](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); + } + }); + }, + td::Timestamp::now()); } void NodeActor::after_db_store_torrent_meta(td::Result R) { diff --git a/storage/storage-daemon/CMakeLists.txt b/storage/storage-daemon/CMakeLists.txt index cabc61439..5392d4ab9 100644 --- a/storage/storage-daemon/CMakeLists.txt +++ b/storage/storage-daemon/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(embed-provider-code smartcont/embed-provider-code.cpp) add_custom_command( diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 98388428d..d1cbf37b5 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -42,6 +42,8 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/port/rlimit.h" + #include namespace ton { @@ -950,6 +952,7 @@ int main(int argc, char *argv[]) { SCOPE_EXIT { td::log_interface = td::default_log_interface; }; + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 786432)); td::IPAddress ip_addr; bool client_mode = false; diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 98b900a1e..44580dd59 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDACTOR_SOURCE td/actor/core/ActorExecutor.cpp diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index c4ff79a1b..662ae20c4 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(BENCHMARK_SOURCE benchmark.cpp third_party/mp-queue.c diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 55476e648..57e69403b 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDDB_UTILS_SOURCE td/db/utils/BlobView.cpp diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 12c3a4f8d..6bbe7df3c 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,19 +20,53 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/Span.h" +#include "td/utils/ThreadSafeCounter.h" #include namespace td { +struct UsageStats { + size_t get_count{}; + size_t get_found_count{}; + size_t get_not_found_count{}; + size_t set_count{}; + UsageStats operator+(const UsageStats& other) const { + return UsageStats{.get_count = get_count + other.get_count, + .get_found_count = get_found_count + other.get_found_count, + .get_not_found_count = get_not_found_count + other.get_not_found_count, + .set_count = set_count + other.set_count}; + } + UsageStats operator-(const UsageStats& other) const { + return UsageStats{.get_count = get_count - other.get_count, + .get_found_count = get_found_count - other.get_found_count, + .get_not_found_count = get_not_found_count - other.get_not_found_count, + .set_count = set_count - other.set_count}; + } + NamedStats to_named_stats() const { + NamedStats ns; + ns.stats_int["usage_get_count"] += get_count; + ns.stats_int["usage_get_found_count"] += get_found_count; + ns.stats_int["usage_get_not_found_count"] += get_not_found_count; + ns.stats_int["usage_set_count"] += set_count; + return ns; + } +}; +inline td::StringBuilder& operator<<(td::StringBuilder& sb, const UsageStats& stats) { + sb << "get: " << stats.get_count << ", +" << stats.get_found_count << ", -" << stats.get_not_found_count; + return sb; +} + class KeyValueReader { public: virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; virtual Result get(Slice key, std::string &value) = 0; + virtual Result> get_multi(td::Span keys, std::vector *values) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } - virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + virtual Status for_each_in_range(Slice begin, Slice end, std::function f) { return td::Status::Error("foreach_range is not supported"); } }; @@ -42,9 +76,17 @@ class PrefixedKeyValueReader : public KeyValueReader { PrefixedKeyValueReader(std::shared_ptr reader, Slice prefix) : reader_(std::move(reader)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return reader_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return reader_->count(PSLICE() << prefix_ << prefix); } @@ -54,14 +96,16 @@ class PrefixedKeyValueReader : public KeyValueReader { std::string prefix_; }; -class KeyValueUtils { - public: -}; - class KeyValue : public KeyValueReader { public: virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status merge(Slice key, Slice value) { + return Status::Error("merge is not supported"); + } + virtual Status run_gc() { + return Status::OK(); + } virtual Status begin_write_batch() = 0; virtual Status commit_write_batch() = 0; @@ -80,14 +124,25 @@ class KeyValue : public KeyValueReader { virtual Status flush() { return Status::OK(); } + virtual UsageStats get_usage_stats() { + return {}; + } }; class PrefixedKeyValue : public KeyValue { public: PrefixedKeyValue(std::shared_ptr kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return kv_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return kv_->count(PSLICE() << prefix_ << prefix); } diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 080133602..c79daccea 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -19,60 +19,114 @@ #include "td/db/MemoryKeyValue.h" #include "td/utils/format.h" +#include "td/utils/Span.h" namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { - auto it = map_.find(key); - if (it == map_.end()) { + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.get_count++; + auto it = map.find(key); + if (it == map.end()) { + usage_stats_.get_not_found_count++; return GetStatus::NotFound; } value = it->second; + usage_stats_.get_found_count++; return GetStatus::Ok; } +std::unique_ptr MemoryKeyValue::lock(td::Slice key) { + auto bucket_id = std::hash()(std::string_view(key.data(), key.size())) % buckets_.size(); + return lock(buckets_[bucket_id]); +} + +Result> MemoryKeyValue::get_multi(td::Span keys, + std::vector *values) { + values->resize(keys.size()); + std::vector res; + res.reserve(keys.size()); + for (size_t i = 0; i < keys.size(); i++) { + res.push_back(get(keys[i], (*values)[i]).move_as_ok()); + } + return res; +} + Status MemoryKeyValue::for_each(std::function f) { - for (auto &it : map_) { - TRY_STATUS(f(it.first, it.second)); + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + for (auto &it : bucket->map) { + TRY_STATUS(f(it.first, it.second)); + } } return Status::OK(); } Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { - for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { - if (it->first < end) { - TRY_STATUS(f(it->first, it->second)); - } else { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(begin); it != map.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } } } return Status::OK(); } Status MemoryKeyValue::set(Slice key, Slice value) { - map_[key.str()] = value.str(); + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.set_count++; + map[key.str()] = value.str(); return Status::OK(); } +Status MemoryKeyValue::merge(Slice key, Slice update) { + CHECK(merger_); + auto bucket = lock(key); + auto &map = bucket->map; + auto &value = map[key.str()]; + merger_->merge_value_and_update(value, update); + if (value.empty()) { + map.erase(key.str()); + } + return td::Status::OK(); +} Status MemoryKeyValue::erase(Slice key) { - auto it = map_.find(key); - if (it != map_.end()) { - map_.erase(it); + auto bucket = lock(key); + auto &map = bucket->map; + auto it = map.find(key); + if (it != map.end()) { + map.erase(it); } return Status::OK(); } Result MemoryKeyValue::count(Slice prefix) { size_t res = 0; - for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) { - if (Slice(it->first).truncate(prefix.size()) != prefix) { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(prefix); it != map.end(); it++) { + if (Slice(it->first).truncate(prefix.size()) != prefix) { + break; + } + res++; } - res++; } return res; } std::unique_ptr MemoryKeyValue::snapshot() { auto res = std::make_unique(); - res->map_ = map_; + for (size_t i = 0; i < buckets_.size(); i++) { + auto bucket = lock(buckets_[i]); + res->buckets_[i].map = bucket->map; + } return std::move(res); } @@ -80,10 +134,10 @@ std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } Status MemoryKeyValue::begin_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::commit_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::abort_write_batch() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index f0b5faa08..6a5be7866 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -22,12 +22,23 @@ #include namespace td { + +struct Merger { + virtual ~Merger() = default; + virtual void merge_value_and_update(std::string &value, Slice update) = 0; + virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; +}; class MemoryKeyValue : public KeyValue { public: - Result get(Slice key, std::string &value) override; + MemoryKeyValue() = default; + MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { + } + Result get(Slice key, std::string& value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; @@ -43,8 +54,30 @@ class MemoryKeyValue : public KeyValue { std::string stats() const override; + UsageStats get_usage_stats() override { + return usage_stats_; + } + private: - std::map> map_; + static constexpr size_t buckets_n = 64; + struct Bucket { + std::mutex mutex; + std::map> map; + }; + struct Unlock { + void operator()(Bucket* bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; int64 get_count_{0}; + UsageStats usage_stats_{}; + std::shared_ptr merger_; + + std::unique_ptr lock(Bucket& bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key); }; } // namespace td diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1aa64a5d..660381a31 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,12 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "rocksdb/filter_policy.h" +#include "td/utils/misc.h" namespace td { namespace { -static Status from_rocksdb(rocksdb::Status status) { +static Status from_rocksdb(const rocksdb::Status &status) { if (status.ok()) { return Status::OK(); } @@ -56,62 +58,92 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { + if (transaction_db_) { + return RocksDb{transaction_db_, options_}; + } return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { - rocksdb::OptimisticTransactionDB *db; - { - rocksdb::Options db_options; + rocksdb::Options db_options; + db_options.merge_operator = options.merge_operator; + db_options.compaction_filter = options.compaction_filter; - static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (!options.no_block_cache && options.block_cache == nullptr) { - options.block_cache = default_cache; - } + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } - rocksdb::BlockBasedTableOptions table_options; - if (options.no_block_cache) { - table_options.no_block_cache = true; - } else { - table_options.block_cache = options.block_cache; + rocksdb::BlockBasedTableOptions table_options; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + if (options.enable_bloom_filter) { + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + if (options.two_level_index_and_filter) { + table_options.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.partition_filters = true; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; } - db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - db_options.use_direct_reads = options.use_direct_reads; - db_options.manual_wal_flush = true; - db_options.create_if_missing = true; - db_options.max_background_compactions = 4; - db_options.max_background_flushes = 2; - db_options.bytes_per_sync = 1 << 20; - db_options.writable_file_max_buffer_size = 2 << 14; - db_options.statistics = options.statistics; - db_options.max_log_file_size = 100 << 20; - db_options.keep_log_file_num = 1; - rocksdb::OptimisticTransactionDBOptions occ_options; - occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + // table_options.block_align = true; + if (options.no_reads) { + db_options.memtable_factory.reset(new rocksdb::VectorRepFactory()); + db_options.allow_concurrent_memtable_write = false; + } + + db_options.wal_recovery_mode = rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; + + if (options.experimental) { + // Place your experimental options here + } + + if (options.no_transactions) { + rocksdb::DB *db{nullptr}; + TRY_STATUS(from_rocksdb(rocksdb::DB::Open(db_options, std::move(path), &db))); + return RocksDb(std::shared_ptr(db), std::move(options)); + } else { + rocksdb::OptimisticTransactionDB *db{nullptr}; rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; + return RocksDb(std::shared_ptr(db), std::move(options)); } - return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { return rocksdb::CreateDBStatistics(); } -std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { +std::string RocksDb::statistics_to_string(const std::shared_ptr &statistics) { return statistics->ToString(); } -void RocksDb::reset_statistics(const std::shared_ptr statistics) { +void RocksDb::reset_statistics(const std::shared_ptr &statistics) { statistics->Reset(); } @@ -133,7 +165,9 @@ std::string RocksDb::stats() const { } Result RocksDb::get(Slice key, std::string &value) { - //LOG(ERROR) << "GET"; + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::Status status; if (snapshot_) { rocksdb::ReadOptions options; @@ -153,6 +187,40 @@ Result RocksDb::get(Slice key, std::string &value) { return from_rocksdb(status); } +Result> RocksDb::get_multi(td::Span keys, std::vector *values) { + std::vector statuses(keys.size()); + std::vector keys_rocksdb; + keys_rocksdb.reserve(keys.size()); + for (auto &key : keys) { + keys_rocksdb.push_back(to_rocksdb(key)); + } + std::vector values_rocksdb(keys.size()); + rocksdb::ReadOptions options; + if (snapshot_) { + options.snapshot = snapshot_.get(); + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else if (transaction_) { + transaction_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else { + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } + std::vector res(statuses.size()); + values->resize(statuses.size()); + for (size_t i = 0; i < statuses.size(); i++) { + auto &status = statuses[i]; + if (status.ok()) { + res[i] = GetStatus::Ok; + values->at(i) = values_rocksdb[i].ToString(); + } else if (status.code() == rocksdb::Status::kNotFound) { + res[i] = GetStatus::NotFound; + values->at(i) = ""; + } else { + return from_rocksdb(status); + } + } + return res; +} + Status RocksDb::set(Slice key, Slice value) { if (write_batch_) { return from_rocksdb(write_batch_->Put(to_rocksdb(key), to_rocksdb(value))); @@ -162,6 +230,18 @@ Status RocksDb::set(Slice key, Slice value) { } return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value))); } +Status RocksDb::merge(Slice key, Slice value) { + if (write_batch_) { + return from_rocksdb(write_batch_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + if (transaction_) { + return from_rocksdb(transaction_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + return from_rocksdb(db_->Merge({}, to_rocksdb(key), to_rocksdb(value))); +} +Status RocksDb::run_gc() { + return from_rocksdb(db_->CompactRange({}, nullptr, nullptr)); +} Status RocksDb::erase(Slice key) { if (write_batch_) { @@ -174,7 +254,11 @@ Status RocksDb::erase(Slice key) { } Result RocksDb::count(Slice prefix) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -197,7 +281,11 @@ Result RocksDb::count(Slice prefix) { } Status RocksDb::for_each(std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -219,7 +307,11 @@ Status RocksDb::for_each(std::function f) { } Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -252,9 +344,10 @@ Status RocksDb::begin_write_batch() { Status RocksDb::begin_transaction() { CHECK(!write_batch_); + CHECK(transaction_db_); rocksdb::WriteOptions options; options.sync = true; - transaction_.reset(db_->BeginTransaction(options, {})); + transaction_.reset(transaction_db_->BeginTransaction(options, {})); return Status::OK(); } @@ -307,7 +400,11 @@ Status RocksDb::end_snapshot() { } RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) - : db_(std::move(db)), options_(options) { + : transaction_db_{db}, db_(std::move(db)), options_(std::move(options)) { +} + +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(std::move(options)) { } void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 499a33281..c9fa93e10 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -36,12 +36,16 @@ #include namespace rocksdb { +class DB; +class Comparator; class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; class Snapshot; class Statistics; +class MergeOperator; +class CompactionFilter; } // namespace rocksdb namespace td { @@ -61,8 +65,18 @@ struct RocksDbOptions { std::shared_ptr statistics = nullptr; std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; + + std::shared_ptr merge_operator = nullptr; + const rocksdb::CompactionFilter *compaction_filter = nullptr; + + bool experimental = false; + bool no_reads = false; + bool no_transactions = false; + bool use_direct_reads = false; bool no_block_cache = false; + bool enable_bloom_filter = false; + bool two_level_index_and_filter = false; }; class RocksDb : public KeyValue { @@ -72,11 +86,14 @@ class RocksDb : public KeyValue { static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; + Status run_gc() override; Result count(Slice prefix) override; Status for_each(std::function f) override; - Status for_each_in_range (Slice begin, Slice end, std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -94,8 +111,8 @@ class RocksDb : public KeyValue { std::string stats() const override; static std::shared_ptr create_statistics(); - static std::string statistics_to_string(const std::shared_ptr statistics); - static void reset_statistics(const std::shared_ptr statistics); + static std::string statistics_to_string(const std::shared_ptr &statistics); + static void reset_statistics(const std::shared_ptr &statistics); static std::shared_ptr create_cache(size_t capacity); @@ -103,12 +120,13 @@ class RocksDb : public KeyValue { RocksDb &operator=(RocksDb &&); ~RocksDb(); - std::shared_ptr raw_db() const { + std::shared_ptr raw_db() const { return db_; }; private: - std::shared_ptr db_; + std::shared_ptr transaction_db_; + std::shared_ptr db_; RocksDbOptions options_; std::unique_ptr transaction_; @@ -123,5 +141,6 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot_; explicit RocksDb(std::shared_ptr db, RocksDbOptions options); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tdfec/CMakeLists.txt b/tdfec/CMakeLists.txt index 828ff90d5..97e4eb691 100644 --- a/tdfec/CMakeLists.txt +++ b/tdfec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDFEC_SOURCE td/fec/raptorq/Rfc.cpp td/fec/raptorq/Rfc.h diff --git a/tdfec/benchmark/CMakeLists.txt b/tdfec/benchmark/CMakeLists.txt index ee8f72cbf..66224a6d4 100644 --- a/tdfec/benchmark/CMakeLists.txt +++ b/tdfec/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(benchmark-fec benchmark.cpp ) target_include_directories(benchmark-fec PUBLIC $) target_link_libraries(benchmark-fec PRIVATE tdfec) diff --git a/tdnet/CMakeLists.txt b/tdnet/CMakeLists.txt index bc00a6769..0da2258d9 100644 --- a/tdnet/CMakeLists.txt +++ b/tdnet/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDNET_SOURCE td/net/FdListener.cpp td/net/TcpListener.cpp @@ -12,7 +10,7 @@ set(TDNET_SOURCE add_library(tdnet STATIC ${TDNET_SOURCE}) target_include_directories(tdnet PUBLIC $) -target_link_libraries(tdnet PUBLIC tdactor) +target_link_libraries(tdnet PUBLIC tdactor ${TUNNEL_LIB_IF_USED}) add_executable(tcp_ping_pong example/tcp_ping_pong.cpp) target_link_libraries(tcp_ping_pong PRIVATE tdactor tdnet) diff --git a/tdnet/td/net/TcpListener.cpp b/tdnet/td/net/TcpListener.cpp index e711cbbda..c9bebe2d4 100644 --- a/tdnet/td/net/TcpListener.cpp +++ b/tdnet/td/net/TcpListener.cpp @@ -61,7 +61,7 @@ void TcpListener::loop() { break; } TRY_RESULT(client_socket, std::move(r_socket)); - LOG(ERROR) << "Accept"; + LOG(INFO) << "Accept"; callback_->accept(std::move(client_socket)); } if (td::can_close(server_socket_fd_)) { diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index ba28c5cfc..cae5c4d51 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -20,7 +20,12 @@ #include "td/net/FdListener.h" #include "td/net/TcpListener.h" +#ifdef TON_USE_GO_TUNNEL +#include "td/net/tunnel/libtunnel.h" +#endif + #include "td/utils/BufferedFd.h" +#include "td/utils/filesystem.h" #include @@ -29,6 +34,172 @@ namespace { int VERBOSITY_NAME(udp_server) = VERBOSITY_NAME(DEBUG) + 10; } namespace detail { + +constexpr int TUNNEL_BUFFER_SZ_PACKETS = 100; +constexpr int TUNNEL_MAX_PACKET_MTU = 1500; +constexpr double TUNNEL_ALARM_EVERY = 0.01; + +class UdpServerTunnelImpl : public UdpServer { + public: + void start_up() override; + void alarm() override; + + void send(td::UdpMessage &&message) override; + static td::actor::ActorOwn create(td::Slice name, std::string global_config, std::string tunnel_config, std::unique_ptr callback, + td::Promise on_ready); + + UdpServerTunnelImpl(std::string global_config, std::string tunnel_config, std::unique_ptr callback, td::Promise on_ready); + +private: + td::Promise on_ready_; + uint8_t out_buf_[(sizeof(sockaddr)+2+TUNNEL_MAX_PACKET_MTU)*TUNNEL_BUFFER_SZ_PACKETS]; + size_t out_buf_offset_ = 0; + size_t out_buf_msg_num_ = 0; + size_t tunnel_index_; + double last_batch_at_ = Time::now(); + + std::string global_config_; + std::string tunnel_config_; + + int32 port_; + std::unique_ptr callback_; + + static void on_recv_batch(void *next, uint8_t *data, size_t num); + static void on_reinit(void *next, sockaddr *addr); + + static void log(const char *text, const size_t len, const int level) { + const string str(text, len); + switch (level) { + case 0: + LOG(FATAL) << "[TUNNEL] " << str; + break; + case 1: + LOG(ERROR) << "[TUNNEL] " << str; + break; + case 2: + LOG(WARNING) << "[TUNNEL] " << str; + break; + case 3: + LOG(INFO) << "[TUNNEL] " << str; + break; + default: + LOG(DEBUG) << "[TUNNEL] " << str; + break; + } + } +}; + +void UdpServerTunnelImpl::send(td::UdpMessage &&message) { + const auto sock = message.address.get_sockaddr(); + const auto sz = message.data.size(); + + // ip+port + memcpy(out_buf_ + out_buf_offset_, sock, sizeof(sockaddr)); + out_buf_offset_ += sizeof(sockaddr); + + // data len (2 bytes) + out_buf_[out_buf_offset_] = static_cast(sz >> 8); + out_buf_[out_buf_offset_ + 1] = static_cast(sz & 0xff); + + if (sz > TUNNEL_MAX_PACKET_MTU) { + LOG(WARNING) << "udp message is too big, dropping"; + return; + } + + memcpy(out_buf_ + out_buf_offset_ + 2, message.data.data(), sz); + out_buf_offset_ += 2 + sz; + out_buf_msg_num_++; + + + if (out_buf_msg_num_ == TUNNEL_BUFFER_SZ_PACKETS) { +#ifdef TON_USE_GO_TUNNEL + WriteTunnel(tunnel_index_, out_buf_, out_buf_msg_num_); + LOG(DEBUG) << "Sending messages by fulfillment " << TUNNEL_BUFFER_SZ_PACKETS; +#endif + + out_buf_offset_ = 0; + out_buf_msg_num_ = 0; + last_batch_at_ = Time::now(); + } +} + +void UdpServerTunnelImpl::alarm() { + if (out_buf_msg_num_ > 0 && Time::now()-last_batch_at_ >= TUNNEL_ALARM_EVERY) { +#ifdef TON_USE_GO_TUNNEL + WriteTunnel(tunnel_index_, out_buf_, out_buf_msg_num_); + LOG(DEBUG) << "Sending messages by alarm " << out_buf_msg_num_; +#endif + + out_buf_offset_ = 0; + out_buf_msg_num_ = 0; + last_batch_at_ = Time::now(); + } + + alarm_timestamp() = td::Timestamp::in(TUNNEL_ALARM_EVERY); +} + +void UdpServerTunnelImpl::start_up() { +#ifdef TON_USE_GO_TUNNEL + auto global_conf_data_R = td::read_file(global_config_); + if (global_conf_data_R.is_error()) { + LOG(FATAL) << global_conf_data_R.move_as_error_prefix("failed to read global config: "); + return; + } + + auto global_cfg = global_conf_data_R.move_as_ok(); + + LOG(INFO) << "Initializing ADNL Tunnel..."; + const auto res = PrepareTunnel(&log, &on_recv_batch, &on_reinit, callback_.get(), callback_.get(), tunnel_config_.data(), tunnel_config_.size(), global_cfg.data(), global_cfg.size()); + if (!res.index) { + // the reason will be displayed in logs from lib part + exit(1); + } + tunnel_index_ = res.index; + LOG(INFO) << "ADNL Tunnel Initialized"; + + td:IPAddress ip; + ip.init_ipv4_port(td::IPAddress::ipv4_to_str(res.ip), static_cast(res.port)).ensure(); + on_ready_.set_value(std::move(ip)); + + alarm_timestamp() = td::Timestamp::in(TUNNEL_ALARM_EVERY); +#else + LOG(FATAL) << "Tunnel was not enabled during node building, rebuild with cmake flag -DTON_USE_GO_TUNNEL=ON"; +#endif +} + +void UdpServerTunnelImpl::on_recv_batch(void *next, uint8_t *data, size_t num) { + for (size_t i = 0; i < num; i++) { + UdpMessage msg{}; + msg.address.init_sockaddr(reinterpret_cast(data)); + const uint16_t len = (static_cast(data[16]) << 8) + static_cast(data[17]); + msg.data = BufferSlice(reinterpret_cast(data + 18), len); + data += 18+len; + + // both init_sockaddr and BufferSlice doing memcpy so it is safe + static_cast(next)->on_udp_message(std::move(msg)); + } +} + +void UdpServerTunnelImpl::on_reinit(void *next, sockaddr *addr) { + td::IPAddress ip; + ip.init_sockaddr(addr); + + static_cast(next)->on_in_addr_update(std::move(ip)); +} + +td::actor::ActorOwn UdpServerTunnelImpl::create(td::Slice name, std::string global_config, std::string tunnel_config, + std::unique_ptr callback, + td::Promise on_ready) { + return td::actor::create_actor( + actor::ActorOptions().with_name(name).with_poll(!td::Poll::is_edge_triggered()), global_config, tunnel_config, std::move(callback), std::move(on_ready)); +} + +UdpServerTunnelImpl::UdpServerTunnelImpl(std::string global_config, std::string tunnel_config, std::unique_ptr callback, td::Promise on_ready): on_ready_(std::move(on_ready)) + , global_config_(global_config) + , tunnel_config_(tunnel_config) + , callback_(std::move(callback)) { +} + class UdpServerImpl : public UdpServer { public: void send(td::UdpMessage &&message) override; @@ -396,6 +567,13 @@ Result> UdpServer::create(td::Slice name, int32 port, fd.maximize_rcv_buffer().ensure(); return detail::UdpServerImpl::create(name, std::move(fd), std::move(callback)); } + +Result> UdpServer::create_via_tunnel(td::Slice name, std::string global_config, std::string tunnel_config, + std::unique_ptr callback, + td::Promise on_ready) { + return detail::UdpServerTunnelImpl::create(name, global_config, tunnel_config, std::move(callback), std::move(on_ready)); +} + Result> UdpServer::create_via_tcp(td::Slice name, int32 port, std::unique_ptr callback) { return actor::create_actor(name, port, std::move(callback)); diff --git a/tdnet/td/net/UdpServer.h b/tdnet/td/net/UdpServer.h index f6d0d268e..e6de2dcc5 100644 --- a/tdnet/td/net/UdpServer.h +++ b/tdnet/td/net/UdpServer.h @@ -28,15 +28,23 @@ namespace td { class UdpServer : public td::actor::Actor { public: class Callback { - public: + public: virtual ~Callback() = default; virtual void on_udp_message(td::UdpMessage udp_message) = 0; }; + class TunnelCallback : public Callback { + public: + virtual void on_in_addr_update(td::IPAddress ip) = 0; + }; virtual void send(td::UdpMessage &&message) = 0; static Result> create(td::Slice name, int32 port, std::unique_ptr callback); static Result> create_via_tcp(td::Slice name, int32 port, std::unique_ptr callback); + static Result> create_via_tunnel(td::Slice name, std::string global_config, std::string tunnel_config, + std::unique_ptr callback, + td::Promise on_ready); + }; } // namespace td diff --git a/tdnet/td/net/tunnel/libtunnel.h b/tdnet/td/net/tunnel/libtunnel.h new file mode 100644 index 000000000..3c6aa31ec --- /dev/null +++ b/tdnet/td/net/tunnel/libtunnel.h @@ -0,0 +1,117 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "lib.go" + +#include +#include + +typedef struct { + size_t index; + int ip; + int port; +} Tunnel; + +// next - is pointer to class instance or callback to call method from node code +typedef void (*RecvCallback)(void* next, uint8_t* data, size_t num); + +typedef void (*ReinitCallback)(void* next, struct sockaddr* data); + +typedef void (*Logger)(const char *text, const size_t len, const int level); + + +// we need it because we cannot call C func by pointer directly from go +static inline void on_recv_batch_ready(RecvCallback cb, void* next, void* data, size_t num) { + cb(next, (uint8_t*)data, num); +} + +static inline void on_reinit(ReinitCallback cb, void* next, void* data) { + cb(next, (struct sockaddr*)data); +} + +static inline void write_log(Logger log, const char *text, const size_t len, const int level) { + log(text, len, level); +} + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + + +//goland:noinspection ALL +extern Tunnel PrepareTunnel(Logger logger, RecvCallback onRecv, ReinitCallback onReinit, void* nextOnRecv, void* nextOnReinit, char* configPath, int configPathLen, char* networkConfigJson, int networkConfigJsonLen); +extern int WriteTunnel(size_t tunIdx, uint8_t* data, size_t num); + +#ifdef __cplusplus +} +#endif diff --git a/tdtl/CMakeLists.txt b/tdtl/CMakeLists.txt index 482bd0f7b..0601b9220 100644 --- a/tdtl/CMakeLists.txt +++ b/tdtl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDTL_SOURCE td/tl/tl_config.cpp diff --git a/tdtl/td/tl/tl_simple.h b/tdtl/td/tl/tl_simple.h index dd5a1e53a..06d6ea133 100644 --- a/tdtl/td/tl/tl_simple.h +++ b/tdtl/td/tl/tl_simple.h @@ -81,12 +81,16 @@ struct Type { struct Arg { const Type *type; std::string name; + int var_num = -1; + int exist_var_num = -1; + int exist_var_bit = -1; }; struct Constructor { std::string name; std::int32_t id; std::vector args; + int var_count = 0; const CustomType *type; }; @@ -100,6 +104,7 @@ struct CustomType { struct Function { std::string name; + int var_count = 0; std::int32_t id; std::vector args; const Type *type; @@ -248,11 +253,15 @@ class Schema { constructor = constructors_.back().get(); constructor->id = from->id; constructor->name = from->name; + constructor->var_count = from->var_count; constructor->type = get_custom_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; constructor->args.push_back(std::move(arg)); } } @@ -266,11 +275,15 @@ class Schema { function = functions_.back().get(); function->id = from->id; function->name = from->name; + function->var_count = from->var_count; function->type = get_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; function->args.push_back(std::move(arg)); } } diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index f8c191a14..e1d0c87b8 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) if (WIN32) @@ -216,6 +214,7 @@ set(TDUTILS_SOURCE td/utils/invoke.h td/utils/JsonBuilder.h td/utils/List.h + td/utils/LRUCache.h td/utils/logging.h td/utils/MemoryLog.h td/utils/misc.h @@ -320,6 +319,10 @@ set(TDUTILS_TEST_SOURCE PARENT_SCOPE ) +if (NOT TON_PRINT_BACKTRACE_ON_CRASH) + set_source_files_properties(td/utils/port/signals.cpp PROPERTIES COMPILE_DEFINITIONS TON_DISABLE_BACKTRACE) +endif() + #RULES #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index 194fda391..391286066 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - # Generates files for MIME type <-> extension conversions # DEPENDS ON: gperf grep bash/powershell diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index 9de11fcae..22f524465 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -291,9 +291,12 @@ void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumCon LOG_IF(FATAL, result != 1); } -void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { +td::Status BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context); - LOG_IF(FATAL, result != r.impl_->big_num); + if (result != r.impl_->big_num) { + return td::Status::Error("Failed to compute modulo inverse"); + } + return td::Status::OK(); } void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, diff --git a/tdutils/td/utils/BigNum.h b/tdutils/td/utils/BigNum.h index b0094659c..682b441ab 100644 --- a/tdutils/td/utils/BigNum.h +++ b/tdutils/td/utils/BigNum.h @@ -109,7 +109,7 @@ class BigNum { static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); - static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); + static td::Status mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, BigNumContext &context); diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index 1380f0291..d40006785 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -23,8 +23,10 @@ #if TD_HAVE_ABSL #include #include +#include #else #include +#include #endif namespace td { @@ -34,11 +36,15 @@ template > using HashMap = absl::flat_hash_map; template , class E = std::equal_to<>> using NodeHashMap = absl::node_hash_map; +template +using BTreeMap = absl::btree_map; #else template > using HashMap = std::unordered_map; template > using NodeHashMap = std::unordered_map; +template +using BTreeMap = std::map; #endif } // namespace td diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h new file mode 100644 index 000000000..d8f51525b --- /dev/null +++ b/tdutils/td/utils/LRUCache.h @@ -0,0 +1,127 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include +#include "List.h" +#include "check.h" + +namespace td { + +template +class LRUCache { + public: + explicit LRUCache(uint64 max_size) : max_size_(max_size) { + } + LRUCache(const LRUCache&) = delete; + LRUCache& operator=(const LRUCache&) = delete; + + V* get_if_exists(const K& key, bool update = true) { + auto it = cache_.find(key); + if (it == cache_.end()) { + return nullptr; + } + Entry* entry = it->get(); + if (update) { + entry->remove(); + lru_.put(entry); + } + return &entry->value; + } + + bool contains(const K& key) const { + return cache_.contains(key); + } + + bool put(const K& key, V value, bool update = true, uint64 weight = 1) { + bool added = false; + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key, std::move(value), weight)).first; + added = true; + total_weight_ += weight; + } else { + (*it)->value = std::move(value); + if (update) { + (*it)->remove(); + } + } + if (update) { + lru_.put(it->get()); + cleanup(); + } + return added; + } + + V& get(const K& key, bool update = true, uint64 weight = 1) { + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key, weight)).first; + total_weight_ += weight; + } else if (update) { + (*it)->remove(); + } + V& result = (*it)->value; + if (update) { + lru_.put(it->get()); + cleanup(); + } + return result; + } + + private: + struct Entry : ListNode { + Entry(K key, uint64 weight) : key(std::move(key)), weight(weight) { + } + Entry(K key, V value, uint64 weight) : key(std::move(key)), value(std::move(value)), weight(weight) { + } + K key; + V value; + uint64 weight; + }; + struct Cmp { + using is_transparent = void; + bool operator()(const std::unique_ptr& a, const std::unique_ptr& b) const { + return a->key < b->key; + } + bool operator()(const std::unique_ptr& a, const K& b) const { + return a->key < b; + } + bool operator()(const K& a, const std::unique_ptr& b) const { + return a < b->key; + } + }; + std::set, Cmp> cache_; + ListNode lru_; + uint64 max_size_; + uint64 total_weight_ = 0; + + void cleanup() { + while (total_weight_ > max_size_ && cache_.size() > 1) { + auto to_remove = (Entry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + total_weight_ -= to_remove->weight; + cache_.erase(cache_.find(to_remove->key)); + } + } +}; + +} // namespace td diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index e6504e358..1a5f8fa36 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -414,7 +414,9 @@ class MpmcQueue { while (true) { auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); auto &block = node->block; - if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + auto read_pos = block.read_pos.load(); + auto write_pos = block.write_pos.load(); + if (write_pos <= read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { return false; } auto pos = block.read_pos++; diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index cff808143..f75de466a 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -619,6 +619,13 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { return status.print(string_builder); } +template +StringBuilder &operator<<(StringBuilder &sb, const Result &result) { + if (result.is_ok()) { + return sb << "Ok{" << result.ok() << "}"; + } + return sb << result.error(); +} namespace detail { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index aa976b2fb..8601275a9 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,13 +19,17 @@ #pragma once -#include "td/utils/common.h" +#include "port/thread.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" #include "td/utils/ThreadLocalStorage.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" #include #include +#include #include namespace td { @@ -69,6 +73,50 @@ class ThreadSafeCounter { ThreadSafeMultiCounter<1> counter_; }; +struct NamedStats { + std::map stats_int; + std::map stats_str; + + NamedStats with_suffix(const std::string &suffix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[p.first + suffix] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[p.first + suffix] = p.second; + } + return res; + } + NamedStats with_prefix(const std::string &prefix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[prefix + p.first] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[prefix + p.first] = p.second; + } + return res; + } + void apply_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] += p.second; + } + for (auto &p : other.stats_str) { + stats_str[p.first] = p.second; + } + } + void subtract_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] -= p.second; + } + } + NamedStats combine_with(const NamedStats &other) const { + NamedStats res = *this; + res.apply_diff(other); + return res; + } +}; + class NamedThreadSafeCounter { static constexpr int N = 128; using Counter = ThreadSafeMultiCounter; @@ -79,6 +127,9 @@ class NamedThreadSafeCounter { CounterRef() = default; CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { } + void inc() { + add(1); + } void add(int64 diff) { counter_->add(index_, diff); } @@ -119,6 +170,11 @@ class NamedThreadSafeCounter { f(names_[i], counter_.sum(i)); } } + NamedStats get_stats() const { + NamedStats res; + for_each([&](Slice name, int64 cnt) { res.stats_int.emplace(name.str(), cnt); }); + return res; + } void clear() { std::unique_lock guard(mutex_); @@ -181,11 +237,11 @@ struct NamedPerfCounter { } // namespace td -#define TD_PERF_COUNTER(name) \ +#define TD_PERF_COUNTER(name) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; -#define TD_PERF_COUNTER_SINCE(name, since) \ +#define TD_PERF_COUNTER_SINCE(name, since) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ - auto scoped_perf_##name = \ + auto scoped_perf_##name = \ td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 24de099aa..abb69b100 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -22,6 +22,9 @@ #include "td/utils/logging.h" #include "td/utils/Time.h" +#include +#include + namespace td { Timer::Timer(bool is_paused) : is_paused_(is_paused) { @@ -60,12 +63,15 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { return string_builder << format::as_time(timer.elapsed()); } -PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function&& callback) +PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function &&callback) : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) { } PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) - : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) { + : name_(std::move(other.name_)) + , start_at_(other.start_at_) + , max_duration_(other.max_duration_) + , callback_(std::move(other.callback_)) { other.start_at_ = 0; } @@ -134,4 +140,34 @@ double ThreadCpuTimer::elapsed() const { return res; } +PerfLogAction PerfLog::start_action(std::string name) { + auto i = entries_.size(); + entries_.push_back({.name = std::move(name), .begin = td::Timestamp::now().at()}); + return PerfLogAction{i, std::unique_ptr(this)}; +} +td::StringBuilder &operator<<(StringBuilder &sb, const PerfLog &log) { + sb << "{"; + std::vector ids(log.entries_.size()); + std::iota(ids.begin(), ids.end(), 0); + std::sort(ids.begin(), ids.end(), [&](auto a, auto b) { + return log.entries_[a].end - log.entries_[a].begin > log.entries_[b].end - log.entries_[b].begin; + }); + sb << "{"; + for (size_t i = 0; i < log.entries_.size(); i++) { + sb << "\n\t"; + auto &entry = log.entries_[ids[i]]; + sb << "{" << entry.name << ":" << entry.begin << "->" << entry.end << "(" << entry.end - entry.begin << ")" + << td::format::cond(entry.status.is_error(), entry.status, "") << "}"; + } + sb << "\n}"; + return sb; +} + +double PerfLog::finish_action(size_t i, td::Status status) { + auto &entry = entries_[i]; + CHECK(entry.end == 0); + entry.end = td::Timestamp::now().at(); + entry.status = std::move(status); + return entry.end - entry.begin; +} } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index a27cac8a7..de793aec0 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/StringBuilder.h" +#include "td/utils/Status.h" #include @@ -46,7 +47,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function &&callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); @@ -80,4 +81,93 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class RealCpuTimer { + public: + RealCpuTimer() : RealCpuTimer(false) { + } + explicit RealCpuTimer(bool is_paused) : real_(is_paused), cpu_(is_paused) { + } + RealCpuTimer(const RealCpuTimer &other) = default; + RealCpuTimer &operator=(const RealCpuTimer &other) = default; + + double elapsed_real() const { + return real_.elapsed(); + } + double elapsed_cpu() const { + return cpu_.elapsed(); + } + struct Time { + double real = 0.0, cpu = 0.0; + double get(bool is_cpu) const { + return is_cpu ? cpu : real; + } + Time &operator+=(const Time &other) { + real += other.real; + cpu += other.cpu; + return *this; + } + Time operator-(const Time &other) const { + return {.real = real - other.real, .cpu = cpu - other.cpu}; + } + }; + Time elapsed_both() const { + return {.real = real_.elapsed(), .cpu = cpu_.elapsed()}; + } + void pause() { + real_.pause(); + cpu_.pause(); + } + void resume() { + real_.resume(); + cpu_.resume(); + } + + private: + Timer real_; + ThreadCpuTimer cpu_; +}; + +class PerfLog; +struct EmptyDeleter { + template + void operator()(T *) { + } +}; +class PerfLogAction { + public: + template + double finish(const T &result); + size_t i_{0}; + std::unique_ptr perf_log_; +}; + +class PerfLog { + public: + PerfLogAction start_action(std::string name); + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const PerfLog &log); + + private: + struct Entry { + std::string name; + double begin{}; + double end{}; + td::Status status; + }; + std::vector entries_; + friend class PerfLogAction; + + double finish_action(size_t i, td::Status status); +}; +template +double PerfLogAction::finish(const T &result) { + if (!perf_log_) { + return 0.0; + } + if (result.is_ok()) { + return perf_log_->finish_action(i_, td::Status::OK()); + } else { + return perf_log_->finish_action(i_, result.error().clone()); + } +} + } // namespace td diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index 30d183abe..cf4447186 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -179,11 +179,12 @@ inline StringBuilder &operator<<(StringBuilder &logger, Time t) { double value; }; - static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, {"s", 1}}; + static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, + {"s", 1}, {"h", 3600}, {"d", 86400}}; static constexpr size_t durations_n = sizeof(durations) / sizeof(NamedValue); size_t i = 0; - while (i + 1 < durations_n && t.seconds_ > 10 * durations[i + 1].value) { + while (i + 1 < durations_n && std::abs(t.seconds_) > 10 * durations[i + 1].value) { i++; } logger << StringBuilder::FixedDouble(t.seconds_ / durations[i].value, 1) << durations[i].name; diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 1d60b2fa6..12962f29e 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -317,7 +317,7 @@ static void block_stdin() { #endif } -static void default_failure_signal_handler(int sig) { +[[maybe_unused]] static void default_failure_signal_handler(int sig) { Stacktrace::init(); signal_safe_write_signal_number(sig); @@ -334,9 +334,11 @@ Status set_default_failure_signal_handler() { Stdin(); // init static variables before atexit #endif std::atexit(block_stdin); +#ifndef TON_DISABLE_BACKTRACE TRY_STATUS(setup_signals_alt_stack()); TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler)); TRY_STATUS(set_signal_handler(SignalType::Error, default_failure_signal_handler)); +#endif return Status::OK(); } diff --git a/tdutils/td/utils/tl_parsers.cpp b/tdutils/td/utils/tl_parsers.cpp index eb78d3e1c..c41cfeb95 100644 --- a/tdutils/td/utils/tl_parsers.cpp +++ b/tdutils/td/utils/tl_parsers.cpp @@ -33,7 +33,8 @@ TlParser::TlParser(Slice slice) { if (data_len <= small_data_array.size() * sizeof(int32)) { buf = &small_data_array[0]; } else { - LOG(ERROR) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " << slice.begin(); + LOG(DEBUG) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " + << (const void *)slice.begin(); data_buf = std::make_unique(1 + data_len / sizeof(int32)); buf = data_buf.get(); } diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index af51153f3..2d0c50304 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 14d5958b2..16db1c9f1 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -16,15 +16,17 @@ Test_Fift_test_dict_default a9c8cbcfdece5573185022cea07f59f1bc404e5d879e5157a574 Test_Fift_test_disasm_default 412cf37d37c5d9d81f44dbf4e3d3e7cda173c23b890614eb8a3bc5f2b92f13e6 Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac54756e76953f0d9afca Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec +Test_Fift_test_get_extra_balance_default cc2428172b660ae66489e1b4868786654195137cc29524022cd1cfcf6708336d Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c +Test_Fift_test_p256_default e1948ddd3d2686baa9f70fdf376ffcebbc2ec5f20eeb366cd856254e61fbfa31 Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a -Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b +Test_Fift_test_tvm_runvm_default b3e0d70c00f0e8bdf6d45c56c7c0817fa5be7a3fc540190786233a394de72e42 Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 0fde1e68d..3ee723ec8 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -303,8 +303,9 @@ class TestNode : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -347,7 +348,7 @@ class TestNode : public td::actor::Actor { } } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } @@ -358,8 +359,8 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -380,9 +381,6 @@ class TestNode : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/tl-utils/CMakeLists.txt b/tl-utils/CMakeLists.txt index d5c52d48a..709bc7cb5 100644 --- a/tl-utils/CMakeLists.txt +++ b/tl-utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TL_UTILS_SOURCE common-utils.hpp tl-utils.hpp diff --git a/tl/CMakeLists.txt b/tl/CMakeLists.txt index d0760a349..168a5d73e 100644 --- a/tl/CMakeLists.txt +++ b/tl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_subdirectory(generate) set_source_files_properties(${TL_TON_API} PROPERTIES GENERATED TRUE) add_library(tl_api STATIC diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 083d39737..8ff7f79a4 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - file(MAKE_DIRECTORY auto/tl) set(TL_TON_API diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index cfc9f3a19..d737dea5a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -232,7 +232,9 @@ overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = ov overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; -overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.nodesV2 nodes:(vector overlay.nodeV2) = overlay.NodesV2; + +overlay.pong = overlay.Pong; overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; @@ -269,7 +271,8 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; -overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; +overlay.getRandomPeersV2 peers:overlay.nodesV2 = overlay.NodesV2; +overlay.ping = overlay.Pong; overlay.query overlay:int256 = True; overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; @@ -340,6 +343,8 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; +validatorSession.optimisticCandidateBroadcast flags:# prev_candidate_id:int256 data:bytes = validatorSession.OptimisticCandidateBroadcast; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -406,6 +411,9 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateIdV2 block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt effective_shard:long = tonNode.PersistentStateIdV2; +tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; +tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; @@ -424,6 +432,8 @@ tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_ha proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; @@ -432,11 +442,18 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; + +// optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments +tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt + limits:ImportedMsgQueueLimits proof:tonNode.OutMsgQueueProof = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; tonNode.customOverlayId zero_state_file_hash:int256 name:string nodes:(vector int256) = tonNode.CustomOverlayId; +tonNode.fastSyncOverlayId zero_state_file_hash:int256 shard:tonNode.shardId = tonNode.FastSyncOverlayId; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -445,6 +462,7 @@ ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; @@ -459,6 +477,8 @@ tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts: tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; +tonNode.newFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = tonNode.NewFastSyncMemberCertificate; + ---functions--- @@ -471,14 +491,11 @@ tonNode.prepareBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool = tonNode.PreparedProof; tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; -tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; @@ -490,12 +507,22 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; +tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; +tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; + +tonNode.downloadPersistentStateSliceV2 state:tonNode.persistentStateIdV2 offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSizeV2 state:tonNode.persistentStateIdV2 = tonNode.PersistentStateSize; + tonNode.getCapabilities = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; tonNode.query = Object; +tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int = overlay.MemberCertificate; + ---types--- // bit 0 - started @@ -537,7 +564,9 @@ db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:i db.filedb.key.empty = db.filedb.Key; db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; -db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key; +db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; @@ -555,6 +584,8 @@ db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_t db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionShard block:tonNode.blockIdExt split_depth:int = db.state.PersistentStateDescriptionShard; +db.state.persistentStateDescriptionShardsV2 shard_blocks:(vector db.state.PersistentStateDescriptionShard) = db.state.PersistentStateDescriptionShards; db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; @@ -626,8 +657,8 @@ liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; -liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteserver.desc id:PublicKey ip:int port:int hostname:string = liteserver.Desc; +liteserver.descV2 id:PublicKey ip:int port:int hostname:string slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; @@ -638,6 +669,7 @@ engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; engine.validatorAdnlAddress id:int256 expire_at:int = engine.ValidatorAdnlAddress; engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs:(vector engine.validatorAdnlAddress) election_date:int expire_at:int = engine.Validator; +engine.collator adnl_id:int256 shard:tonNode.shardId = engine.Collator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; @@ -647,10 +679,16 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.fastSyncOverlayClient adnl_id:int256 slot:int = engine.validator.FastSyncOverlayClient; +engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; +engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) + collator_node_whitelist:engine.validator.collatorNodeWhitelist + fast_sync_overlay_clients:(vector engine.validator.fastSyncOverlayClient) = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) - validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) + validators:(vector engine.validator) collators:(vector engine.collator) + fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) fullnodeconfig:engine.validator.fullNodeConfig extraconfig:engine.validator.extraConfig @@ -660,14 +698,25 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) - = engine.validator.CustomOverlay; + skip_public_msg_send:Bool = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int - whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; + whitelist:(vector string) prioritylist:(vector string) + force_full_collated_data:Bool ignore_collated_data_limits:Bool = engine.validator.CollatorOptions; + +engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator; +engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) + self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard; +engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList; + +engine.validator.shardBlockVerifierConfig.shard shard_id:(tonNode.shardId) trusted_nodes:(vector int256) + required_confirms:int = engine.validator.shardBlockVerifierConfig.Shard; +engine.validator.shardBlockVerifierConfig shards:(vector engine.validator.shardBlockVerifierConfig.shard) + = engine.validator.ShardBlockVerifierConfig; ---functions--- ---types--- @@ -712,7 +761,8 @@ engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckt engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int - traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode; + traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic + last_ping_at:double last_ping_time:double = engine.validator.OverlayStatsNode; engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) @@ -724,6 +774,8 @@ engine.validator.shardOverlayStats.neighbour id:string verison_major:int version roundtrip:double unreliability:double = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string active:Bool neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; +engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) + member_certificate:overlay.MemberCertificate receive_broadcasts:Bool = engine.validator.FastSyncOverlayStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; @@ -733,6 +785,11 @@ engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSiz engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; +engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string banned_for:double = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) + collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; +engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; ---functions--- @@ -803,6 +860,26 @@ engine.validator.getActorTextStats = engine.validator.TextStats; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; +engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; + +engine.validator.collatorNodeSetWhitelistedValidator adnl_id:int256 add:Bool = engine.validator.Success; +engine.validator.collatorNodeSetWhitelistEnabled enabled:Bool = engine.validator.Success; +engine.validator.showCollatorNodeWhitelist = engine.validator.CollatorNodeWhitelist; + +engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; +engine.validator.clearCollatorsList = engine.validator.Success; +engine.validator.showCollatorsList = engine.validator.CollatorsList; +engine.validator.getCollationManagerStats = engine.validator.CollationManagerStats; + +engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; +engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; +engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; +engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; + +engine.validator.setShardBlockVerifierConfig config:engine.validator.shardBlockVerifierConfig = engine.validator.Success; +engine.validator.showShardBlockVerifierConfig = engine.validator.ShardBlockVerifierConfig; + ---types--- storage.pong = storage.Pong; @@ -849,38 +926,97 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int - limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; - -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 - comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double - collation_time:double collated_at:double collation_cached:Bool - collation_work_time:double collation_cpu_work_time:double - collation_stats:validatorSession.collationStats - validation_time:double validated_at:double validation_cached:Bool - validation_work_time:double validation_cpu_work_time:double - gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string - signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string - serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; - -validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; - -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int - creator:int256 total_validators:int total_weight:long - signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long - first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; - -validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int - last_key_block_seqno:int timestamp:double - self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; - -validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; -validatorSession.endValidatorGroupStats session_id:int256 timestamp:double - nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; +validatorStats.stats.producer flags:# + validator_id:int256 block_status:int + candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256 + is_accepted:flags.0?Bool is_ours:flags.0?Bool + got_block_at:flags.0?double got_block_by:flags.0?int got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string + collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256 + validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool + approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string + signed_weight:flags.0?long signed_33pct_at:flags.0?double signed_66pct_at:flags.0?double signers:flags.0?string + serialize_time:flags.4?double deserialize_time:flags.4?double serialized_size:flags.4?int = validatorStats.stats.Producer; + +validatorStats.stats.round started_at:double producers:(vector validatorStats.stats.producer) = validatorStats.stats.Round; + +validatorStats.stats flags:# + session_id:int256 self:int256 block_id:tonNode.blockIdExt cc_seqno:int success:Bool timestamp:double + creator:flags.0?int256 + total_validators:int total_weight:long + signatures:flags.0?int signatures_weight:flags.0?long + approve_signatures:flags.0?int approve_signatures_weight:flags.0?long + first_round:int rounds:(vector validatorStats.stats.round) = validatorStats.Stats; + +validatorStats.blockLimitsStatus + bytes:int gas:int lt_delta:int collated_data_bytes:int + cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + load_fraction_queue_cleanup:double load_fraction_dispatch:double + load_fraction_internals:double load_fraction_externals:double + load_fraction_new_msgs:double + limits_log:string = validatorStats.BlockLimitsStatus; +validatorStats.blockStats.extMsgsStats + ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; +validatorStats.blockStats.neighborStats shard:tonNode.shardId is_trivial:Bool is_local:Bool msg_limit:int + processed_msgs:int skipped_msgs:int limit_reached:Bool = validatorStats.blockStats.NeighborStats; +validatorStats.blockStats + ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) + old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int + neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; +validatorStats.storageStatCacheStats + small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; + +validatorStats.collatedBlock + block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double + bytes:int collated_data_bytes:int attempt:int + self:int256 is_validator:Bool + total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string + block_limits:validatorStats.blockLimitsStatus + block_stats:validatorStats.blockStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; + +validatorStats.validatedBlock + block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double + self:int256 + valid:Bool comment:string + bytes:int collated_data_bytes:int + total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string + storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; + +validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; +validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int + last_key_block_seqno:int started_at:double prev:(vector tonNode.blockIdExt) + self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; + +validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; +validatorStats.endValidatorGroup session_id:int256 timestamp:double self:int256 + nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup; + +validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:double + block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256 + = validatorStats.CollatorNodeResponse; + +collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; +collatorNode.pong#5bbf0521 flags:# version:flags.0?int = collatorNode.Pong; +collatorNode.error code:int message:string = collatorNode.Error; + +shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; +shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlockVerifier.ConfirmBlocks; ---functions--- +collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.generateBlockOptimistic shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.requestBlockCallback flags:# block_id:tonNode.BlockIdExt = collatorNode.Candidate; +collatorNode.ping flags:# = collatorNode.Pong; + +shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96ecb7751..bf531b7af 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 31ca6fd40..e7d8cb59c 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -57,7 +57,7 @@ raw.message hash:bytes source:accountAddress destination:accountAddress value:in raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; -raw.extMessageInfo hash:bytes = raw.ExtMessageInfo; +raw.extMessageInfo hash:bytes hash_norm:bytes = raw.ExtMessageInfo; pchan.config alice_public_key:string alice_address:accountAddress bob_public_key:string bob_address:accountAddress init_timeout:int32 close_timeout:int32 channel_id:int64 = pchan.Config; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 10b9ed8db..9b366407e 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tl/generate/tl_json_converter.cpp b/tl/generate/tl_json_converter.cpp index a0213ef51..d1a0488fb 100644 --- a/tl/generate/tl_json_converter.cpp +++ b/tl/generate/tl_json_converter.cpp @@ -48,13 +48,30 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he sb << " {\n"; sb << " auto jo = jv.enter_object();\n"; sb << " jo(\"@type\", \"" << constructor->name << "\");\n"; + std::vector var_names(constructor->var_count); + for (auto &arg : constructor->args) { + if (arg.var_num >= 0) { + CHECK(arg.var_num < (int)var_names.size()); + var_names[arg.var_num] = tl::simple::gen_cpp_field_name(arg.name); + } + } for (auto &arg : constructor->args) { auto field_name = tl::simple::gen_cpp_field_name(arg.name); - // TODO: or as null - bool is_custom = arg.type->type == tl::simple::Type::Custom; - - if (is_custom) { - sb << " if (object." << field_name << ") {\n "; + bool is_optional = arg.type->type == tl::simple::Type::Custom || arg.exist_var_num >= 0; + + if (is_optional) { + sb << " if ("; + if (arg.type->type == tl::simple::Type::Custom) { + sb << "object." << field_name; + if (arg.exist_var_num >= 0) { + sb << " && "; + } + } + if (arg.exist_var_num >= 0) { + CHECK(arg.exist_var_num < (int)var_names.size()); + sb << "(object." << var_names[arg.exist_var_num] << " & " << (1 << arg.exist_var_bit) << ")"; + } + sb << ") {\n "; } auto object = PSTRING() << "object." << tl::simple::gen_cpp_field_name(arg.name); if (arg.type->type == tl::simple::Type::Bytes || arg.type->type == tl::simple::Type::SecureBytes) { @@ -72,7 +89,7 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he object = PSTRING() << "JsonVectorInt64{" << object << "}"; } sb << " jo(\"" << arg.name << "\", ToJson(" << object << "));\n"; - if (is_custom) { + if (is_optional) { sb << " }\n"; } } diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a-tests.tolk similarity index 81% rename from tolk-tester/tests/a6.tolk rename to tolk-tester/tests/a-tests.tolk index 944945464..fb553f7e5 100644 --- a/tolk-tester/tests/a6.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -4,19 +4,19 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { __expect_type(D, "int"); __expect_type(D*D, "int"); __expect_type(calc_phi, "() -> int"); - return (Dx/D,Dy/D); + return (Dx/D,Dy/D,) };;;; fun calc_phi(): int { var n = 1; - repeat (70) { n*=10; }; + repeat (70) { n*=10 }; var p= 1; var `q`=1; _=`q`; do { - (p,q)=(q,p+q); + (p,q)=(q,p+q) } while (q <= n); //;; - return mulDivRound(p, n, q); + return mulDivRound(p, n, q,) } fun calc_sqrt2(): int { @@ -31,10 +31,10 @@ fun calc_sqrt2(): int { return mulDivRound(p, n, q); } -fun calc_root(m: int) { +fun calc_root(m: int,) { var base: int=1; repeat(70) { base *= 10; } - var (a, b, c) = (1,0,-m); + var (a, b, c) = (1,0,-m,); var (p1, q1, p2, q2) = (1, 0, 0, 1); do { var k: int=-1; @@ -43,7 +43,7 @@ fun calc_root(m: int) { k+=1; (a1, b1, c1) = (a, b, c); c+=b; - c += b += a; + c += b += a } while (c <= 0); (a, b, c) = (-c1, -b1, -a1); (p1, q1) = (k * p1+q1, p1); @@ -67,17 +67,17 @@ fun ataninv(base: int, q: int): int { // computes base*atan(1/q) fun calc_pi(): int { var base: int = 64; - repeat (70) { base *= 10; } + repeat (70) { base *= 10 } return (ataninv(base << 2, 5) - ataninv(base, 239))~>>4; } fun main(): int { - return calc_pi(); + return calc_pi() } /** - method_id | in | out + method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 -@code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 +@code_hash 85818713521853656486584648797214567489479452958868213061669627117427586818086 */ diff --git a/tolk-tester/tests/a6_1.tolk b/tolk-tester/tests/a6_1.tolk deleted file mode 100644 index 8079972b1..000000000 --- a/tolk-tester/tests/a6_1.tolk +++ /dev/null @@ -1,22 +0,0 @@ -fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { - var D: int = a * d - b * c; - var Dx: int = e * d - b * f; - var Dy: int = a * f - e * c; - return (Dx / D, Dy / D); -} - -@method_id(101) -fun testDivMod(x: int, y: int) { - return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); -} - -/** - method_id | in | out -@testcase | 0 | 1 1 1 -1 10 6 | 8 2 -@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 -@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 -@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 -@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 -@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 -@testcase | 101 | 112 3 | 37 1 1 37 33 6 -*/ diff --git a/tolk-tester/tests/a6_5.tolk b/tolk-tester/tests/a6_5.tolk deleted file mode 100644 index 43fd59c5a..000000000 --- a/tolk-tester/tests/a6_5.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@deprecated -fun twice(f: int -> int, x: int) { - return f (f (x)); -} - -fun sqr(x: int) { - return x * x; -} - -fun main(x: int): int { - var f = sqr; - return twice(f, x) * f(x); -} - -@method_id(4) -fun pow6(x: int): int { - return twice(sqr, x) * sqr(x); -} - -/** - method_id | in | out -@testcase | 0 | 3 | 729 -@testcase | 0 | 10 | 1000000 -@testcase | 4 | 3 | 729 -@testcase | 4 | 10 | 1000000 -*/ diff --git a/tolk-tester/tests/a7.tolk b/tolk-tester/tests/a7.tolk deleted file mode 100644 index 1c0ae2eb3..000000000 --- a/tolk-tester/tests/a7.tolk +++ /dev/null @@ -1,24 +0,0 @@ -fun main() { } -@method_id(1) -fun steps(x: int): int { - var n = 0; - while (x > 1) { - n += 1; - if (x & 1) { - x = 3 * x + 1; - } else { - x >>= 1; - } - } - return n; -} - -/** - method_id | in | out -@testcase | 1 | 1 | 0 -@testcase | 1 | 2 | 1 -@testcase | 1 | 5 | 5 -@testcase | 1 | 19 | 20 -@testcase | 1 | 27 | 111 -@testcase | 1 | 100 | 25 -*/ diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow-post-modification.tolk similarity index 81% rename from tolk-tester/tests/allow_post_modification.tolk rename to tolk-tester/tests/allow-post-modification.tolk index e20e8218e..f825d12f8 100644 --- a/tolk-tester/tests/allow_post_modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -1,19 +1,25 @@ fun unsafe_tuple(x: X): tuple asm "NOP"; +@noinline fun inc(x: int, y: int): (int, int) { return (x + y, y * 10); } -fun `~inc`(mutate self: int, y: int): int { +@noinline +fun int.`~inc`(mutate self, y: int): int { val (newX, newY) = inc(self, y); self = newX; return newY; } +@noinline fun eq(v: X): X { return v; } +@noinline fun eq2(v: (int, int)) { return v; } +@noinline fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } -fun multens(mutate self: (int, int), v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } +@noinline +fun (int, int).multens(mutate self, v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } @method_id(11) fun test_return(x: int): (int, int, int, int, int, int, int) { @@ -38,6 +44,7 @@ fun test_tuple_assign(x: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } +@noinline fun foo1(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } @@ -47,7 +54,8 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) { return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); } -fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) { +@noinline +fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) { var (x3: int, x4: int, x5: int, x6: int) = x3456; return (x1, x2, x3, x4, x5, x6, x7); } @@ -102,14 +110,24 @@ fun test_assign_with_mutate(x: int) { @method_id(23) fun test_assign_tensor(x: (int, int)) { var fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); } global fs: (int, int); @method_id(24) fun test_assign_tensor_global(x: (int, int)) { fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); +} + +struct Point1D { + c: int; +} + +@method_id(25) +fun test_with_single_slot_struct(x: Point1D) { + var result = (x, x.c += 10, [x, x.c += 20, eq(x.c -= 50), x.c], eq2((x.c, x.c *= eq(x.c /= 2)))); + return result; } fun main() { @@ -132,20 +150,19 @@ fun main() { @testcase | 22 | 100 | 100 210 4200 630 @testcase | 23 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 @testcase | 24 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 +@testcase | 25 | 100 | 100 110 [ 110 130 80 80 ] 80 3200 @fif_codegen """ - ~inc PROC:<{ - // self y - inc CALLDICT // self newY + int.~inc() PROC:<{ // self y + inc() CALLDICT // self newY }> """ @fif_codegen """ - test_assign_tensor_global PROC:<{ - // x.0 x.1 + test_assign_tensor_global() PROC:<{ // x.0 x.1 """ -@code_hash 61280273714870328160131559159866470128402169974050439159015534193532598351244 +@code_hash 77620375659834063567341916666636335765114427483375544778177892407598637223147 */ diff --git a/tolk-tester/tests/annotations-tests.tolk b/tolk-tester/tests/annotations-tests.tolk new file mode 100644 index 000000000..730b0ee0b --- /dev/null +++ b/tolk-tester/tests/annotations-tests.tolk @@ -0,0 +1,31 @@ +@deprecated("anything") +@custom("anything") +global a: int; + +@deprecated +@custom +const ASDF = 1; + +@custom("props", {allowed: false, ids: [1,2,3]}) +@deprecated +fun f() {} + +@custom({ + `type`: 123, + value: 19 +}) +@custom("another", 12, "annotation") +struct TTT {} + +@deprecated +type MyMsg = int; + +@custom(1,2,3,4) +@custom({function: "main", contract: self}) +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 + */ diff --git a/tolk-tester/tests/asm_arg_order.tolk b/tolk-tester/tests/asm-arg-order.tolk similarity index 57% rename from tolk-tester/tests/asm_arg_order.tolk rename to tolk-tester/tests/asm-arg-order.tolk index b96e09ecb..c42c1f958 100644 --- a/tolk-tester/tests/asm_arg_order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -2,7 +2,7 @@ fun empty_tuple2(): tuple asm "NIL"; @pure -fun tpush2(mutate self: tuple, x: X): void +fun tuple.tpush2(mutate self, x: X): void asm "TPUSH"; @pure @@ -15,11 +15,14 @@ asm (z y x -> 0) "3 TUPLE"; fun asm_func_3(x: int, y: int, z: int): tuple asm (y z x -> 0) "3 TUPLE"; @pure +fun int.asm_func_3(self, y: int, z: int): tuple +asm (y z self -> 0) "3 TUPLE"; +@pure fun asm_func_4(a: int, b: (int, (int, int)), c: int): tuple asm (b a c -> 0) "5 TUPLE"; @pure -fun asm_func_modify(mutate self: tuple, b: int, c: int): void +fun tuple.asm_func_modify(mutate self, b: int, c: int): void asm (c b self) "SWAP TPUSH SWAP TPUSH"; global t: tuple; @@ -117,6 +120,77 @@ fun test_new_dot(): (tuple, tuple) { return (t, t2); } +@pure +fun asmAPlus1TimesB(a: int, b: int): int + asm(b a) "1 ADDCONST MUL"; + +@pure +fun int.plus1TimesB(self, b: int): int + asm(b self) "1 ADDCONST MUL"; + +@pure +@noinline +fun get2Pure() { return 2; } +@pure +@noinline +fun get10Pure() { return 10; } + +@noinline +fun get2Impure() { return 2; } +@noinline +fun get10Impure() { return 10; } + +global g2: int; +global g10: int; + +fun setG2(v: int) { g2 = v; return v; } + +@method_id(27) +fun test27() { + return asmAPlus1TimesB(2, 10); +} + +@method_id(28) +fun test28() { + return asmAPlus1TimesB(get2Pure(), get10Pure()); +} + +@method_id(29) +fun test29() { + return asmAPlus1TimesB(get2Impure(), get10Impure()); +} + +@method_id(30) +fun test30() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2, g10); +} + +@method_id(31) +fun test31() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2 += 2, g10 += g2); +} + +@method_id(32) +fun test32() { + return 2.plus1TimesB(10); +} + +@method_id(33) +fun test33(x: int) { + return ((x += 10).plus1TimesB(2), (x += 20).plus1TimesB(x), ((x /= (g2=2)).plus1TimesB(x*g2)), setG2(7).plus1TimesB(g2)); +} + +@method_id(34) +fun test34() { + var cs = stringHexToSlice("020a"); + return asmAPlus1TimesB(cs.loadUint(8), cs.loadUint(8)); +} + + fun main() { } @@ -134,6 +208,72 @@ fun main() { @testcase | 24 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] @testcase | 25 | | [ 22 33 ] [ 220 330 ] @testcase | 26 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 27 | | 30 +@testcase | 28 | | 30 +@testcase | 29 | | 30 +@testcase | 30 | | 30 +@testcase | 31 | | 70 +@testcase | 32 | | 30 +@testcase | 33 | 0 | 22 930 480 56 +@testcase | 34 | | 30 + +@fif_codegen +""" + test27() PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test28() PROC:<{ + get10Pure() CALLDICT + get2Pure() CALLDICT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test29() PROC:<{ + get2Impure() CALLDICT + get10Impure() CALLDICT + SWAP + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test30() PROC:<{ + ... + $g10 GETGLOB + $g2 GETGLOB + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test32() PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + x{020a} PUSHSLICE + 8 LDU + 8 PLDU + SWAP + 1 ADDCONST MUL + }> +""" -@code_hash 93068291567112337250118419287631047120002003622184251973082208096953112184588 +@code_hash 38717859169035491454494966803813683158800226121601078320213761069394202390827 */ diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index bb6476521..0343bdf50 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -1,11 +1,11 @@ fun extractFromTypedTuple(params: [int]) { - var [payload: int] = params; + var [payload: int, ] = params; return payload + 10; } @method_id(101) fun test101(x: int) { - var params = [x]; + var params = [x, ]; return extractFromTypedTuple(params); } @@ -21,7 +21,7 @@ fun typesAsIdentifiers(builder: builder) { { var cell: cell = cell; var tuple: tuple = createEmptyTuple(); - var bool: bool = tuple.tupleAt(0) > 0; + var bool: bool = tuple.get(0) > 0; } return int; } @@ -29,37 +29,37 @@ fun typesAsIdentifiers(builder: builder) { global callOrder: tuple; fun getTensor_12() { - callOrder.tuplePush(100); + callOrder.push(100); return (1, 2); } fun getTensor_1X(x: int) { - callOrder.tuplePush(101); + callOrder.push(101); return (1, x); } fun getTuple_12() { - callOrder.tuplePush(110); - return [1, 2]; + callOrder.push(110); + return [1, 2, ]; } fun getTuple_1X(x: int) { - callOrder.tuplePush(111); + callOrder.push(111); return [1, x]; } fun getUntypedTuple_12() { - callOrder.tuplePush(120); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(2); + callOrder.push(120); + var t = createEmptyTuple(); t.push(1); t.push(2); return t; } fun getUntypedTuple_1X(x: int) { - callOrder.tuplePush(121); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(x); + callOrder.push(121); + var t = createEmptyTuple(); t.push(1); t.push(x); return t; } fun getIntValue5() { - callOrder.tuplePush(10); + callOrder.push(10); return 5; } fun getIntValueX(x: int) { - callOrder.tuplePush(11); + callOrder.push(11); return x; } @@ -111,7 +111,7 @@ global t107: (int, int); @method_id(107) fun test107() { - ((t107 = (1, 2)).0, (t107 = (3, 4)).1) = (5, 6); + ((t107 = (1, 2)).0, (t107 = (3, 4)).1, ) = (5, 6, ); return t107; } @@ -133,7 +133,7 @@ fun test108() { fun test109() { callOrder = createEmptyTuple(); var x = 0; - [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)]; + [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x), ]; return (callOrder, x); } @@ -168,7 +168,7 @@ fun test112() { @method_id(113) fun test113() { - var (a, t, z) = (1, [2,3], (-1,-1)); + var (a, t, z, ) = (1, [2,3], (-1,-1)); (a, t, a, z, t.1, z.1) = (10, [a,12], 13, (a, t.1), 14, t.1); return (a, t, z); } @@ -203,10 +203,64 @@ fun test116() { return (a, b, c, d, rhs2); } +@method_id(117) +fun test117() { + var c = [((1, ), ), ]; + __expect_type(c, "[int]"); + c.0 = 10; + c.0 = (20, ); + return c; +} + +struct Storage { + owner: slice; +} + +@method_id(118) +fun test118(x: int) { + var i1: int; + if (10 > 3) { + i1 = 10; + } else { { (_, i1) = (5, 20); } } + + var i2: int10; + match (x) { + 1 => i2 = 1, + 2 => i2 = 2, + 3 => i2 = 3, + else => i2 = 4, + } + + var st: Storage?; + if (x > 100) { + throw 123; + } else { + if (x < -100) { throw 456; } + else { st = { owner: "" }} + } + + var i3: (int, int) | builder; + match (i1) { + int => i3 = (1, 2), + } + + var unused: int8; + + return (i1, i2, st.owner.remainingBitsCount(), i3.0); +} + +@method_id(119) +fun test119(a: int, b: int) { + var t: tuple = [a + 1, b * 2] as tuple; + val l = t.size(); + val [c,d:int] = [t.0 as int, t.1]; + return (l, c, d); +} + -fun main(value: int) { - var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); +fun main(value: int, ) { + var (x: int?, y,) = (autoInferIntNull(value), autoInferIntNull(value * 2)); if (x == null && y == null) { return null; } return x == null || y == null ? -1 : x! + y!; } @@ -231,20 +285,22 @@ fun main(value: int) { @testcase | 114 | | 13 [ 1 14 ] 1 3 @testcase | 115 | | [ 101 111 ] 9 9 @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] +@testcase | 117 | | [ 20 ] +@testcase | 118 | 3 | 10 3 0 1 +@testcase | 119 | 1 2 | 2 2 4 @fif_codegen """ - test116 PROC:<{ - // - 1 PUSHINT // '10=1 - 2 PUSHINT // '10=1 '11=2 - 3 PUSHINT // '10=1 '11=2 '12=3 - 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 - 4 TUPLE // rhs - DUP // rhs rhs - 4 UNTUPLE // rhs2 a b c d - 4 ROLL // a b c d rhs2 + test116() PROC:<{ // + 1 PUSHINT // '10=1 + 2 PUSHINT // '10=1 '11=2 + 3 PUSHINT // '10=1 '11=2 '12=3 + 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 + 4 TUPLE // rhs + DUP // rhs rhs + 4 UNTUPLE // rhs2 a b c d + 4 ROLL // a b c d rhs2 }> """ */ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index b01068839..c734740df 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -88,6 +88,7 @@ fun testBoolNegateOptimized(x: bool) { return (x, !x, !!x, !!!x, !!!!true); } +@noinline fun eqX(x: bool) { return x; } @method_id(19) @@ -125,19 +126,17 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - boolWithBitwiseConst PROC:<{ - // - 0 PUSHINT // '3 - -1 PUSHINT // '3 '5 - 0 PUSHINT // '3 '5 '7 - -1 PUSHINT // '3 '5 '7 '8 + boolWithBitwiseConst() PROC:<{ + 0 PUSHINT // '3 + -1 PUSHINT // '3 '5 + 0 PUSHINT // '3 '5 '7 + -1 PUSHINT // '3 '5 '7 '8 }> """ @fif_codegen """ - testDoUntilCodegen PROC:<{ - // i n + testDoUntilCodegen() PROC:<{ // i n 0 PUSHINT // i n cnt=0 UNTIL:<{ INC // i n cnt @@ -163,8 +162,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testConstNegateCodegen PROC:<{ - // + testConstNegateCodegen() PROC:<{ TRUE // '0 FALSE // '0 '1 FALSE // '0 '1 '2 @@ -176,28 +174,26 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolNegateOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 - OVER // x '1 x - NOT // x '1 '2 + testBoolNegateOptimized() PROC:<{ // x + DUP // x x + NOT // x '1 + OVER // x '1 x + NOT // x '1 '2 s2 s(-1) PUXC - TRUE // x '1 x '2 '3 + TRUE // x '1 x '2 '3 }> """ @fif_codegen """ - testBoolCompareOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 + testBoolCompareOptimized() PROC:<{ // x + DUP // x x + NOT // x '1 OVER // x '1 x - eqX CALLDICT // x '1 '2 - NOT // x '1 '3 + eqX() CALLDICT // x '1 '2 + NOT // x '1 '3 s2 PUSH // x '1 '3 x - eqX CALLDICT // x '1 '3 '4 + eqX() CALLDICT // x '1 '3 '4 s3 PUSH // x '1 '3 '4 x }> """ diff --git a/tolk-tester/tests/build-addr-tests.tolk b/tolk-tester/tests/build-addr-tests.tolk new file mode 100644 index 000000000..788450a55 --- /dev/null +++ b/tolk-tester/tests/build-addr-tests.tolk @@ -0,0 +1,310 @@ +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +struct WalletStorage { + balance: coins; + ownerAddress: address; + minterAddress: address; +} + +fun WalletStorage.generateEmptyData(ownerAddress: address, minterAddress: address) { + val emptyWalletStorage: WalletStorage = { + balance: 0, + ownerAddress, + minterAddress, + }; + return emptyWalletStorage.toCell(); +} + +@inline_ref +fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { + var sb = options.closeTo as slice; + sb.skipBits(3); // addr_std$10 + anycast 0 + val wc_b = sb.loadInt(8); + val shardPrefix = sb.loadUint(options.fixedPrefixLength); + + var sa = a as slice; + sa.skipBits(3 + 8 + options.fixedPrefixLength); + + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + anycast 0 + .storeInt(wc_b, 8) + .storeUint(shardPrefix, options.fixedPrefixLength) + .storeSlice(sa); +} + +@method_id(101) +fun test1() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return dd1.endCell().beginParse() as address + == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + +@method_id(102) +fun test2() { + val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + assert((a as slice).remainingBitsCount() == 267, 267); + assert((b as slice).remainingBitsCount() == 267, 267); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return address.fromValidBuilder(dd2) + == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +} + +@method_id(103) +fun test3() { + var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); + var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); + var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); + var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); + return ( + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + ) +} + +@method_id(104) +fun test4(shardDepth: int) { + var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; +} + +@method_id(105) +fun test5() { + var b = beginCell() + .storeUint(0b100, 3) // std addr no anycast + .storeInt(MASTERCHAIN, 8) + .storeUint(0xFFFF, 256); + val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); + return address.fromValidBuilder(b) == a; +} + +@method_id(106) +fun test6() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + + +fun manual_buildAddressOfJettonWallet_plain(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeUint(stateInitHash, 256) +} + +fun manual_buildAddressOfJettonWallet_sharded(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + var mask = stateInitHash & ((1 << (256 - SHARD_DEPTH)) - 1); + val shard_prefix = (ownerAddress as slice).getMiddleBits(3 + 8, SHARD_DEPTH); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeSlice(shard_prefix) + .storeUint( mask, 256 - SHARD_DEPTH); +} + +fun address.manual_isAddressOfJettonWallet_plain(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val (wc, hash) = self.getWorkchainAndHash(); + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +fun address.manual_isAddressOfJettonWallet_sharded(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + var stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val mask = (1 << (256 - SHARD_DEPTH)) - 1; + stateInitHash = stateInitHash & mask; + var (wc, hash) = self.getWorkchainAndHash(); + hash = hash & mask; + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +@method_id(110) +fun test10() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }.addressMatches(checkedAddr) + ) +} + +@method_id(111) +fun test11() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + AutoDeployAddress { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + }.addressMatches(checkedAddr), + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }.addressMatches(checkedAddr) + ) +} + +@method_id(112) +fun test12() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + var i = 1; + while (i < 10) { + var params: AutoDeployAddress = { + workchain: -1, + stateInit: { code: jwCode, data: createEmptyCell() }, + toShard: { fixedPrefixLength: i, closeTo: ownerAddress } + }; + val addrBuilt = params.buildAddress(); + val addr = address.fromValidBuilder(addrBuilt); + assert (params.addressMatches(addr)) throw 123; + params.workchain = 0; + assert (!params.addressMatches(addr)) throw 123; + params.workchain = -1; + params.toShard!.fixedPrefixLength += 1; + assert (!params.addressMatches(addr)) throw 123; + i += 1 + } + return i; +} + +@method_id(113) +fun test13() { + val stateInitCell = StateInit { + fixedPrefixLength: null, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(0, 8) + .storeUint(stateInitCell.hash(), 256); + val si: AutoDeployAddress = { + stateInit: stateInitCell + }; + val addrBuilt = si.buildAddress(); + assert (manualBuilt.endCell().hash() == addrBuilt.endCell().hash()) throw 123; + + val addr = addrBuilt.endCell().beginParse() as address; + return ( + si.addressMatches(addr), + AutoDeployAddress {workchain: 99, stateInit: stateInitCell}.addressMatches(addr), + ) +} + +@method_id(114) +fun test14() { + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val stateInitCell = StateInit { + fixedPrefixLength: 10, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(9, 8) + .storeSlice((ownerAddress as slice).getMiddleBits(3+8, 10)) + .storeUint(stateInitCell.hash() & ((1 << 246) - 1), 246); + val si: AutoDeployAddress = { + workchain: 9, + stateInit: stateInitCell, + toShard: { fixedPrefixLength: 10, closeTo: ownerAddress } + }; + val addrBuilt = si.buildAddress(); + val addr = addrBuilt.endCell().beginParse() as address; + + val stateInitHash = stateInitCell.hash(); + val (wc, hash) = addr.getWorkchainAndHash(); + val manual_isAddrOfContract = ((stateInitHash & ((1 << 246) - 1)) == (hash & ((1 << 246) - 1))) & (9 == wc); + + return ( + manualBuilt.endCell().hash() == addrBuilt.endCell().hash(), + manual_isAddrOfContract, + si.addressMatches(addr), + ) +} + + + +fun main() {} + +/** +@testcase | 101 | | -1 +@testcase | 102 | | -1 +@testcase | 103 | | 39876 24338 15241 50719 11252 15241 +@testcase | 104 | 2 | 24338 +@testcase | 104 | 9 | 39876 +@testcase | 105 | | -1 +@testcase | 106 | | -1 +@testcase | 110 | | 0 0 +@testcase | 111 | | 0 0 0 +@testcase | 112 | | 10 +@testcase | 113 | | -1 0 +@testcase | 114 | | -1 -1 -1 + */ diff --git a/tolk-tester/tests/bytesN-tests.tolk b/tolk-tester/tests/bytesN-tests.tolk new file mode 100644 index 000000000..7f1a76e2f --- /dev/null +++ b/tolk-tester/tests/bytesN-tests.tolk @@ -0,0 +1,48 @@ +fun takeAnySlice(slice: slice) {} +fun getSomeSlice() { return beginCell().endCell().beginParse(); } +fun getNullableBytes16(): bytes16? { return getSomeSlice() as bytes16; } + +const someBytes = "asdf" as bytes16; +const anotherBytes: bytes16 = "asdf" as bytes16; + +fun autoInferBytes16() { + if (random.uint256()) { return someBytes; } + else if (random.uint256()) { return true ? "" as bytes16 : anotherBytes; } + else { return someBytes!; } +} + +@method_id(101) +fun test1() { + var ss = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse() as bits32; + __expect_type(ss, "bits32"); + __expect_type(ss as slice, "slice"); + __expect_type(ss as slice?, "slice?"); + __expect_type(ss as bytes128, "bytes128"); + __expect_type(someBytes, "bytes16"); + __expect_type(10>3 ? (null, ss as bits1) : (ss as bytes1, null), "(bytes1?, bits1?)"); + return ((ss as slice).remainingBitsCount(), (ss as slice).loadInt(32), (ss as slice).loadInt(32)); +} + +fun test2(a: bytes8, b: bits64) { + a = b as bytes8; + b = a as bits64; + (a as slice).loadRef(); + (b as slice?)!.loadRef(); +} + +@method_id(103) +fun test3() { + var slice = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse(); + var bq = slice as bits16?; + if (bq != null) { + return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).remainingBitsCount(); + } + return -1; +} + +fun main() {} + +/** +@testcase | 101 | | 64 1 2 +@testcase | 103 | | 33 + */ diff --git a/tolk-tester/tests/c2.tolk b/tolk-tester/tests/c2.tolk deleted file mode 100644 index bcbc6c938..000000000 --- a/tolk-tester/tests/c2.tolk +++ /dev/null @@ -1,28 +0,0 @@ -global op: (int, int) -> int; - -fun check_assoc(a: int, b: int, c: int): bool { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun unnamed_args(_: int, _: slice, _: int) { - return true; -} - -fun main(x: int, y: int, z: int): bool? { - op = `_+_`; - if (0) { return null; } - return check_assoc(x, y, z); -} - -@method_id(101) -fun test101(x: int, z: int) { - return unnamed_args(x, "asdf", z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -@testcase | 101 | 1 10 | -1 -*/ diff --git a/tolk-tester/tests/c2_1.tolk b/tolk-tester/tests/c2_1.tolk deleted file mode 100644 index ef1e589ad..000000000 --- a/tolk-tester/tests/c2_1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun check_assoc(op: (int, int) -> int, a: int, b: int, c: int) { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun main(x: int, y: int, z: int): bool { - return check_assoc(`_+_`, x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -*/ diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk new file mode 100644 index 000000000..9405c9ead --- /dev/null +++ b/tolk-tester/tests/calls-tests.tolk @@ -0,0 +1,237 @@ +const ZERO = 0; + +fun sumABDef0(a: int, b: int = 0) { + return a + b; +} + +fun sumADef1BDef2(a: int = ((1)), b: int = 2+ZERO) { + return a + b; +} + +fun getAOrB(a: T, b: T, getA: bool = false): T { + return getA ? a : b; +} + +fun pushToTuple(mutate t: tuple, v: T? = null) { + t.push(v); +} + +struct Point { + x: int; + y: int; +} + +fun Point.create(x: int = 0, y: int = 0): Point { + return {x,y}; +} + +fun Point.incrementX(mutate self, deltaX: int = 1) { + self.x += deltaX; +} + +fun makeCost(cost: coins = ton("0.05")) { + return cost + ton("0.05"); +} + +fun makeUnion(v: int | slice = 0) { + return v; +} + +struct MyOptions { + negate: bool = false, + mulBy: int = 0, +} + +global t105: tuple; + +fun log105(a: int, options: MyOptions = {}) { + if (options.negate) { + a = -a; + } + if (options.mulBy) { + a *= options.mulBy; + } + t105.push(a); +} + +struct AnotherOptions { + c1: int; + leaveSign: bool = true; +} + +fun helper106(a: int, options: AnotherOptions = { c1: 1 }) { + a *= options.c1; + return options.leaveSign ? +a : -a; +} + + +@method_id(101) +fun test1(a: int) { + return ( + sumABDef0(a), + sumABDef0(a, 5), + sumABDef0(100), + sumADef1BDef2(a), + sumADef1BDef2(), + sumADef1BDef2(a, 100), + sumADef1BDef2(200, 100), + ) +} + +@method_id(102) +fun test2(a: int) { + var t = createEmptyTuple(); + pushToTuple(mutate t, getAOrB(sumADef1BDef2(a), sumADef1BDef2(a, a))); + pushToTuple(mutate t, true); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + return t; +} + +@method_id(103) +fun test3() { + var p = Point.create(); + p.incrementX(); + var p2 = Point.create(8); + p2.incrementX(p.x += 1); + return (p, p2); +} + +@method_id(104) +fun test4() { + return ( + makeCost(), + makeCost(ton("0.1")), + makeUnion() is int, + makeUnion(), + makeUnion(makeCost()), + ) +} + +@method_id(105) +fun test5() { + t105 = createEmptyTuple(); + log105(1); + log105(1, { negate: true }); + log105(1, { mulBy: 100 }); + log105(1, { mulBy: 100, negate: true }); + log105(1, {}); + return t105; +} + +@method_id(106) +fun test6(l4: bool) { + return ( + helper106(5), + helper106(5, {leaveSign: false, c1: 1}), + helper106(5, {c1: 10}), + helper106(5, {leaveSign: l4, c1: 10}), + helper106(5, {c1: 0}), + ); +} + +fun int.plus(self, v: int = 1) { + return self + v; +} + +@method_id(107) +fun test7() { + return (10.plus(5.plus()), int.plus(4)); +} + +fun createTFrom(v: (int, (U, V)) = (1, (2, 3))) { + return [v.0, v.1.0, v.1.1]; +} + +@method_id(108) +fun test8() { + __expect_type(createTFrom, "((int, (coins, int8))) -> [int, coins, int8]"); + return (createTFrom(), createTFrom((5, (8, createTFrom().0)))); +} + +fun int(initial: int) { return initial } +fun int.create0() { return 0 } +fun int.plus1(self) { return self + 1 } + +@method_id(109) +fun test9() { + return int(10) + int(5).plus1() + int.create0() + int.create0().plus1(); +} + +@noinline fun cmp1(a: int, b: int) { return a == b } +@noinline fun cmp2(a: int, b: int) { return b == a } +@noinline fun cmp3(a: int, b: int) { return a != b } +@noinline fun cmp4(a: int, b: int) { return b != a } + +@method_id(110) +fun test10(a: int, b: int) { + return b - a +} + +@noinline fun leq1(a: int, b: int) { return b < a } +@noinline fun leq2(a: int, b: int) { return b <= a } +@noinline fun leq3(a: int, b: int) { return b > a } +@noinline fun leq4(a: int, b: int) { return b >= a } + +@method_id(111) +fun test11(a: int, b: int) { + return ((b < a), (b <= a), (b > a), (b >= a)); +} + +fun main() {} + +/** +@testcase | 101 | 10 | 10 15 100 12 3 110 300 +@testcase | 102 | 10 | [ 20 -1 (null) (null) (null) ] +@testcase | 103 | | 2 0 10 0 +@testcase | 104 | | 100000000 150000000 -1 0 1 100000000 1 +@testcase | 105 | | [ 1 -1 100 -100 1 ] +@testcase | 106 | 0 | 5 -5 50 -50 0 +@testcase | 107 | | 16 5 +@testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] +@testcase | 109 | | 17 +@testcase | 110 | 5 8 | 3 +@testcase | 111 | 5 8 | 0 0 -1 -1 +@testcase | 111 | 7 7 | 0 -1 0 -1 + +@fif_codegen +""" + cmp1() PROC:<{ + EQUAL + }> + cmp2() PROC:<{ + EQUAL + }> + cmp3() PROC:<{ + NEQ + }> + cmp4() PROC:<{ + NEQ + }> +""" + +@fif_codegen +""" + test10() PROC:<{ + SUBR + }> +""" + +@fif_codegen +""" + leq1() PROC:<{ + GREATER + }> + leq2() PROC:<{ + GEQ + }> + leq3() PROC:<{ + LESS + }> + leq4() PROC:<{ + LEQ + }> +""" + + */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 772812eb1..57a92f03b 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,15 +1,20 @@ -fun store_u32(mutate self: builder, value: int): self { +fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } - -fun load_u32(mutate self: slice): int { +fun slice.load_u32(mutate self): int { return self.loadUint(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; +@pure +fun endCell(b: builder): cell + asm "ENDC"; +@pure +fun beginParse(c: cell): slice + asm "CTOS"; @method_id(101) fun test1(): [int,int,int,int,int] { @@ -71,7 +76,8 @@ fun test5(): [int,int] { @method_id(106) fun test6() { - return beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32); + var st = beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32).endCell().beginParse(); + return st.loadUint(96) == ((1 << 64) + (2 << 32) + 3); } @method_id(107) @@ -80,7 +86,7 @@ fun test7() { var uri_builder = beginCell(); var uri_slice = uri_builder.storeSlice(".json").endCell().beginParse(); var image_slice = uri_builder.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } @method_id(108) @@ -90,13 +96,13 @@ fun test8() { var uri_slice = fresh.storeSlice(".json").endCell().beginParse(); var fresh redef = uri_builder; var image_slice = fresh.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } -fun sumNumbersInSlice(mutate self: slice): int { +fun slice.sumNumbersInSlice(mutate self): int { var result = 0; - while (!self.isEndOfSliceBits()) { + while (!self.isEndOfBits()) { result += self.loadUint(32); } return result; @@ -106,27 +112,27 @@ fun sumNumbersInSlice(mutate self: slice): int { fun test10() { var ref = beginCell().storeInt(100, 32).endCell(); var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); - var result = (getRemainingBitsCount(s), s.sumNumbersInSlice(), getRemainingBitsCount(s), isEndOfSlice(s), isEndOfSliceBits(s), isEndOfSliceRefs(s)); + var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEmpty(), s.isEndOfBits(), s.isEndOfRefs()); var ref2: cell = s.loadRef(); var s2: slice = ref2.beginParse(); - s.assertEndOfSlice(); - return (result, s2.loadInt(32), s2.isEndOfSlice()); + s.assertEnd(); + return (result, s2.loadInt(32), s2.isEmpty()); } @method_id(111) fun test11() { var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).storeInt(6, 32).storeInt(7, 32).endCell().beginParse(); - var size1 = getRemainingBitsCount(s); + var size1 = s.remainingBitsCount(); s!.skipBits(32); var s1: slice = s.getFirstBits(64); var n1 = s1.loadInt(32); - var size2 = getRemainingBitsCount(s); + var size2 = s.remainingBitsCount(); s.loadInt(32); - var size3 = getRemainingBitsCount(s); + var size3 = s.remainingBitsCount(); s.removeLastBits(32); - var size4 = getRemainingBitsCount(s); + var size4 = s.remainingBitsCount(); var n2 = s.loadInt(32); - var size5 = getRemainingBitsCount(s); + var size5 = s.remainingBitsCount(); return (n1, n2, size1, size2, size3, size4, size5); } @@ -134,13 +140,13 @@ fun test11() { fun test12() { var (result1, result2) = (0, 0); try { - beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEndOfSlice(); + beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEnd(); result1 = 100; } catch (code) { result1 = code; } try { - beginCell().endCell().beginParse().assertEndOfSlice(); + beginCell().endCell().beginParse().assertEnd(); result2 = 100; } catch (code) { result2 = code; @@ -153,12 +159,12 @@ fun test13() { var ref2 = beginCell().storeInt(1, 32).endCell(); var ref1 = beginCell().storeInt(1, 32).storeRef(ref2).endCell(); var c = beginCell().storeInt(444, 32).storeRef(ref1).storeRef(ref1).storeRef(ref1).storeRef(ref2).storeInt(4, 32).endCell(); - var (n_cells1, n_bits1, n_refs1) = c.calculateCellSizeStrict(10); + var (n_cells1, n_bits1, n_refs1) = c.calculateSizeStrict(10); var s = c.beginParse(); s.loadRef(); s.loadRef(); var n = s.loadInt(32); - var (n_cells2, n_bits2, n_refs2) = s.calculateSliceSizeStrict(10); + var (n_cells2, n_bits2, n_refs2) = s.calculateSizeStrict(10); return ([n_cells1, n_bits1, n_refs1], [n_cells2, n_bits2, n_refs2], n); } @@ -177,7 +183,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if (s.addressIsNone()) { + if ((s as address).isNone()) { s.skipBits(2); } if (s.loadBool() == false) { @@ -187,11 +193,246 @@ fun test111() { var op2 = s.loadMessageOp(); var q2 = s.loadMessageQueryId(); s.skipBits(64); - s.assertEndOfSlice(); - assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; + s.assertEnd(); return (op1, q1, op2, q2); } +global g8: int; +global g10: int; +global g16: int; +global g32: int; + +@inline_ref +fun initGlobals() { + g8 = 8; + g10 = 10; + g16 = 16; + g32 = 32; +} + +@method_id(116) +fun test16() { + initGlobals(); + var b1 = beginCell().storeUint(g8, g16).storeUint(0xFF, g32).storeUint(g8, g16 * 2); + var b2 = beginCell().storeUint(8, 16).storeUint(0xFF, 32).storeUint(8, 16 * 2); + assert(b1.bitsCount() == b2.bitsCount(), 400); + var c1 = b1.endCell().beginParse(); + var c2 = b2.endCell().beginParse(); + assert(c1.bitsEqual(c2), 400); + assert(c1.loadUint(g16) == g8, 400); + assert(c1.loadUint(g32) == 0xFF, 400); + assert(c1.loadUint(2 * g16) == 8, 400); + return b1; + // 00140008000000ff00000008 + // 00140008000000ff00000008 +} + + +@method_id(117) +fun test17() { + var b = beginCell().storeUint(1, 4).storeCoins(0).storeInt(123, 8); + var s = b.endCell().beginParse(); + return (s.loadUint(4), s.loadCoins(), s.loadUint(8)); +} + +@method_id(118) +fun test18() { + var x = 0; + var b = beginCell(); + b = b.storeUint(x, 14); + x += 12; + if (10 > 3) { x += x; } + if (true) { + b.storeInt(x + 2, 8).storeUint(x = match (x) { 24 => 5, else => 0 }, 4); + } + var s = b.endCell().beginParse(); + return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); +} + +fun test19() { + // numbers with potential overflow for STU are not joined, check via codegen + var b = beginCell(); + b.storeInt(123, 4).storeUint(0xFF, 8).storeUint(0xFF, 8).storeInt(-1, 8); + return b; +} + +@method_id(120) +fun test20() { + var x = false; + var n = 4; + var b = true ? beginCell() : null; + b.storeBool(true).storeBool(x); + b = b.storeBool(true).storeUint(0, n *= 2).storeBool(!!true).storeCoins(0); + var s = b.endCell().beginParse(); + return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); +} + +fun test21(s: slice) { + // successive skipBits are also joined + var x = 8; + s.skipBits(x); + x -= 4; + s = s.skipBits(x).skipBits(2); + x *= 0; + s = s.skipBits(x); + return s; +} + +@method_id(122) +fun test22() { + // different builders aren't mixed, store inside them are joined independently + var (b1, b2) = (beginCell(), beginCell()); + b1.storeUint(8, 16).storeUint(8, 8); + b2.storeUint(8, 32).storeUint(1<<88, 100); + return ( + b1.endCell().beginParse().remainingBitsCount(), + b2.endCell().beginParse().skipBits(32).loadUint(100), + ); +} + +@method_id(123) +fun test23(uns: bool) { + // corner values, signed/unsigned 255/256 + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100).storeInt(3, 55).storeInt(0, 1); + b.storeUint(8, 256); + } else { + b.storeInt(1, 10).storeUint(2, 190).storeInt(3, 54).storeUint(1, 1); + } + return b.bitsCount(); +} + +@method_id(124) +fun test24(uns: bool) { + // doesn't fit into a single STI/STU instruction, is splitted + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100) + .storeInt(3, 100).storeInt(8, 19); + return b.endCell().beginParse().skipBits(200+100).loadInt(19); + } else { + b.storeInt(1, 20).storeUint(2, 200).storeInt(3, 35) + .storeUint(1, 1).storeUint(5, 5).storeUint(10, 10); + return b.endCell().beginParse().skipBits(255+6).loadUint(10); + } +} + +fun builder.my_storeRef(mutate self, value: cell): self + asm(self value) "STREFR"; + +fun builder.my_storeAddress(mutate self, value: address): self + asm(value self) "STSLICE"; + +fun builder.my_storeBuilder(mutate self, value: builder): self + asm(value self) "STB"; + +fun my_PUSHINT_SWAP_STU(mutate b: builder): void + asm "5 PUSHINT" "SWAP" "8 STU"; + + +@noinline +fun demo30(value: cell, dest: builder) { + // SWAP + STREFR => STREF + return dest.my_storeRef(value); +} + +@noinline +fun demo31(dest: builder, value: cell) { + // SWAP + STREF => STREFR + return dest.storeRef(value); +} + +@noinline +fun demo32(dest: builder, value: builder) { + // SWAP + STB => STBR + return dest.my_storeBuilder(value); +} + +@noinline +fun demo33(value: builder, dest: builder) { + // SWAP + STBR => STB + return dest.storeBuilder(value); +} + +@noinline +fun demo34(dest: builder, value: address) { + // SWAP + STSLICE => STSLICER + return dest.my_storeAddress(value); +} + +@noinline +fun demo35(value: slice, dest: builder) { + // SWAP + STSLICER => STSLICE + return dest.storeSlice(value); +} + +@noinline +fun demo36(x: int) { + var b = beginCell(); + if (x > 0) { + // inside IF: N PUSHINT + SWAP + L STU => N PUSHINT + L STUR + my_PUSHINT_SWAP_STU(mutate b); + } + return b; +} + +@noinline +fun demo37(op: int32, u: int8 | int256) { + var b = beginCell(); + // b.storeUint(input.op, 32); + if (u is int8) { + // inside IF: SWAP + 0 PUSHINT + 3 STUR => 0 PUSHINT + 3 STU + b.storeUint(0, 3); + b.storeUint(u, 8); + } + return b; +} + +@method_id(130) +fun test30() { + return (demo37(10, 88 as int8), demo36(50)); +} + +@method_id(131) +fun test31() { + var s = createAddressNone(); + var dest = beginCell().storeUint(3, 32); + assert(demo35(s as slice, dest).endCell().hash() == demo34(dest, s).endCell().hash()) throw 300; + var r1 = demo35(s as slice, dest).endCell().hash(); + var b = beginCell().storeUint(0xFF, 16); + assert(demo33(b, dest).endCell().hash() == demo32(dest, b).endCell().hash()) throw 301; + var r2 = demo33(b, dest).endCell().hash(); + var r = createEmptyCell(); + assert(demo30(r, dest).endCell().hash() == demo31(dest, r).endCell().hash()) throw 302; + var r3 = demo30(r, dest).endCell().hash(); + return [r1 & 0xFFFFFFFF, r2 & 0xFFFFFFFF, r3 & 0xFFFFFFFF]; +} + +@method_id(132) +fun test32(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeUint(x, 32); // SWAP + n STU => n STUR + return b; +} + +@method_id(133) +fun test33(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, 8); // SWAP + n STI => n STIR + return b; +} + +@method_id(134) +fun test34(p: int, n: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, n); // no changes, it's STIX + return b; +} + fun main(): int { return 0; } @@ -202,6 +443,7 @@ fun main(): int { @testcase | 103 | 103 | 103 @testcase | 104 | | [ 1 3 ] @testcase | 105 | | [ 210 1 ] +@testcase | 106 | | -1 @testcase | 107 | | 72 40 72 @testcase | 108 | | 0 40 32 @testcase | 110 | | 64 3 0 0 -1 0 100 -1 @@ -211,22 +453,179 @@ fun main(): int { @testcase | 114 | -1 | -1 0 -1 @testcase | 114 | 0 | 0 0 0 @testcase | 115 | | 123 456 123 456 +@testcase | 116 | | BC{00140008000000ff00000008} +@testcase | 117 | | 1 0 123 +@testcase | 118 | | 0 26 5 +@testcase | 120 | | -1 0 -1 0 -1 0 +@testcase | 122 | | 24 309485009821345068724781056 +@testcase | 123 | -1 | 512 +@testcase | 123 | 0 | 255 +@testcase | 124 | -1 | 8 +@testcase | 124 | 0 | 10 +@testcase | 130 | | BC{00030b10} BC{000205} +@testcase | 131 | | [ 225345949 901620178 1560646286 ] +@testcase | 132 | 0 | BC{00080000000a} +@testcase | 133 | 0 | BC{00020a} +@testcase | 134 | 0 8 | BC{00020a} + +We test that consequtive storeInt/storeUint with constants are joined into a single number + +@fif_codegen +""" + test6() PROC:<{ + NEWC + x{000000010000000200000003} STSLICECONST + ENDC // '11 +""" -Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. @fif_codegen """ - test6 PROC:<{ - // - NEWC // '0 - 1 PUSHINT // '0 '1=1 - SWAP // '1=1 '0 - 32 STU // '0 - 2 PUSHINT // '0 '4=2 - SWAP // '4=2 '0 - 32 STU // '0 - 3 PUSHINT // '0 '7=3 - SWAP // '7=3 '0 - 32 STU // '0 + test17() PROC:<{ + NEWC + x{107b} STSLICECONST +""" + +@fif_codegen +""" + test18() PROC:<{ + NEWC + b{00000000000000000110100101} STSLICECONST +""" + +@fif_codegen +""" + test19() PROC:<{ + 123 PUSHINT + NEWC + 4 STI + x{ffff} STSLICECONST + -1 PUSHINT + 8 STIR + }> +""" + +@fif_codegen +""" + test20() PROC:<{ + NEWC + x{a010} STSLICECONST +""" + +@fif_codegen +""" + test21() PROC:<{ + 14 LDU + NIP + }> +""" + +@fif_codegen +""" + test22() PROC:<{ + NEWC // '2 + NEWC // b1 b2 + SWAP // b2 b1 + x{000808} STSLICECONST // b2 b1 + SWAP // b1 b2 + x{000000080010000000000000000000000} STSLICECONST // b1 b2 +""" + +@fif_codegen +""" + test23() PROC:<{ + NEWC + SWAP + IF:<{ + x{0000000000000000000000001000000000000000000000000200000000000006} STSLICECONST + 8 PUSHINT + 256 STUR + }>ELSE<{ + b{000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000111} STSLICECONST + }> + BBITS + }> +""" + +@fif_codegen +""" + demo30() PROC:<{ // value dest + STREF // dest }> + demo31() PROC:<{ // dest value + STREFR // dest + }> + demo32() PROC:<{ // dest value + STBR // dest + }> + demo33() PROC:<{ // value dest + STB // dest + }> + demo34() PROC:<{ // dest value + STSLICER // dest + }> + demo35() PROC:<{ // value dest + STSLICE // dest + }> +""" + +@fif_codegen """ + demo36() PROC:<{ // x + NEWC // x b + SWAP // b x + 0 GTINT // b '4 + IF:<{ // b + 5 PUSHINT + 8 STUR // b + }> // b + }> +""" + +@fif_codegen +""" + demo37() PROC:<{ + NEWC + s3 POP + 42 EQINT + IF:<{ + SWAP + b{000} STSLICECONST + 8 STU // b + }>ELSE<{ + DROP // b + }> + }> +""" + +@fif_codegen +""" + test32() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 32 STUR // b + }> +""" + +@fif_codegen +""" + test33() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 8 STIR // b + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + NEWC + s0 s2 XCHG + 10 ADDCONST + -ROT + STIX + }> +""" + */ diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/co1.tolk deleted file mode 100644 index f124e1de8..000000000 --- a/tolk-tester/tests/co1.tolk +++ /dev/null @@ -1,72 +0,0 @@ -const int1 = 1; -const int2 = 2; - -const int101: int = 101; -const int111: int = 111; - -const int1r = int1; - -const str1 = "const1"; -const str2 = "aabbcc"s; - -const str2r: slice = str2; - -const str1int = 0x636f6e737431; -const str2int = 0xAABBCC; - -const nibbles: int = 4; - -fun iget1(): int { return int1; } -fun iget2(): int { return int2; } -fun iget3(): int { return int1+int2; } - -fun iget1r(): int { return int1r; } - -fun sget1(): slice { return str1; } -fun sget2(): slice { return str2; } -fun sget2r(): slice { return str2r; } - -const int240: int = ((int1+int2)*10)<<3; - -fun iget240(): int { return int240; } - -@pure -fun newc(): builder -asm "NEWC"; -@pure -fun endcs(b: builder): slice -asm "ENDC" "CTOS"; -@pure -fun sdeq(s1: slice, s2: slice): int -asm "SDEQ"; -@pure -fun stslicer(b: builder, s: slice): builder -asm "STSLICER"; - -fun main() { - var i1: int = iget1(); - var i2: int = iget2(); - var i3: int = iget3(); - - assert(i1 == 1) throw int101; - assert(i2 == 2) throw 102; - assert(i3 == 3) throw 103; - - var s1: slice = sget1(); - var s2: slice = sget2(); - var s3: slice = newc().stslicer(str1).stslicer(str2r).endcs(); - - assert(sdeq(s1, newc().storeUint(str1int, 12 * nibbles).endcs())) throw int111; - assert(sdeq(s2, newc().storeUint(str2int, 6 * nibbles).endcs())) throw 112; - assert(sdeq(s3, newc().storeUint(0x636f6e737431AABBCC, 18 * nibbles).endcs())) throw 113; - - var i4: int = iget240(); - assert(i4 == 240) throw ((104)); - return 0; -} - -/** -@testcase | 0 | | 0 - -@code_hash 61273295789179921867241079778489100375537711211918844448475493726205774530743 -*/ diff --git a/tolk-tester/tests/code_after_ifelse.tolk b/tolk-tester/tests/code_after_ifelse.tolk deleted file mode 100644 index 6a16262f8..000000000 --- a/tolk-tester/tests/code_after_ifelse.tolk +++ /dev/null @@ -1,41 +0,0 @@ -fun elseif(cond: int) { - if (cond > 0) { - throw(cond); - } -} - -@inline -@method_id(101) -fun foo(x: int): int { - if (x==1) { - return 111; - } else { - x *= 2; - } - return x + 1; -} - -fun main(x: int): (int, int) { - return (foo(x), 222); -} - -@method_id(102) -fun test2(x: int) { - try { - if (x < 0) { return -1; } - elseif (x); - } catch(excNo) { - return excNo * 1000; - } - return 0; -} - -/** - method_id | in | out -@testcase | 0 | 1 | 111 222 -@testcase | 0 | 3 | 7 222 -@testcase | 101 | 1 | 111 -@testcase | 101 | 3 | 7 -@testcase | 102 | -5 | -1 -@testcase | 102 | 5 | 5000 -*/ diff --git a/tolk-tester/tests/codegen_check_demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk similarity index 94% rename from tolk-tester/tests/codegen_check_demo.tolk rename to tolk-tester/tests/codegen-check-demo.tolk index 5b46c0935..dec38a885 100644 --- a/tolk-tester/tests/codegen_check_demo.tolk +++ b/tolk-tester/tests/codegen-check-demo.tolk @@ -33,8 +33,7 @@ Below, I just give examples of @fif_codegen tag: @fif_codegen """ -main PROC:<{ - // s +main() PROC:<{ // s 17 PUSHINT // s '3=17 OVER // s z=17 t WHILE:<{ @@ -50,7 +49,7 @@ main PROC:<{ @fif_codegen """ -main PROC:<{ +main() PROC:<{ ... WHILE:<{ ... @@ -77,14 +76,13 @@ main PROC:<{ @fif_codegen """ -test1 PROC:<{ -// +test1() PROC:<{ FALSE }> """ @fif_codegen NOT // '8 -@fif_codegen main PROC:<{ +@fif_codegen main() PROC:<{ @fif_codegen_avoid PROCINLINE @fif_codegen_avoid END c diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk new file mode 100644 index 000000000..96caefe92 --- /dev/null +++ b/tolk-tester/tests/comments-tests.tolk @@ -0,0 +1,42 @@ + +struct WithDef { + f1: int, + f2: int = 10, + f3: int, + f4: int, +} + +const C_20 = 20; + +fun demo_fields_def(x: int) { + var w: WithDef = { + f1: x + C_20, + f3: 100, + f4: C_20 + }; + return (w.f4 + 5, w.f3, w.f2, w.f1); +} + +fun main() { + return 1; +} + +/** +@testcase | 0 | | 1 + +@fif_codegen_enable_comments +@fif_codegen +""" + demo_fields_def() PROC:<{ // x + // 13: f1: x + C_20 + 20 ADDCONST // '6 + 10 PUSHINT // '6 '7=10 + // 14: f3: 100 + 100 PUSHINT // w.f1 w.f2=10 w.f3=100 + // 17: return (w.f4 + 5, w.f3, w.f2, w.f1) + 25 PUSHINT // w.f1 w.f2=10 w.f3=100 '11 + s2 s3 XCHG2 // '11 w.f3=100 w.f2=10 w.f1 + }> +""" + +*/ \ No newline at end of file diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk new file mode 100644 index 000000000..5a386cd4b --- /dev/null +++ b/tolk-tester/tests/constants-tests.tolk @@ -0,0 +1,130 @@ +type MInt = int +type MSlice = slice + +const int1 = 1 +const int2 = 2 + +const int101: int = 101; +const int111: MInt = 111; + +const int1r = int1; + +const str1 = "const1"; +const str2: MSlice = stringHexToSlice("aabbcc"); + +const str2r: slice = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const nibbles: int = 4; + +const strange_zero = (!10 as int); +const strange_minus_1: MInt = (!0 as int); + +const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); + +// since `;` is not mandatory, this is correct from a syntax point of view +const true1 = true const true2 = !!true const true3 = true1 && true2 + +const false1 = !true; +const false2 = false1 || false; + +const tens1 = (1, 2); +const tens2: (int, int, (int8, int16)) = (tens1.1, tens1.0 << 2, tens1); + +const intOrN: int? = null; +const int32Or64: int32 | int64 = 7 as int64; + +fun iget1(): int { return int1; } +fun iget2(): int { return int2; } +fun iget3(): int { return int1+int2; } + +fun iget1r(): int { return int1r; } + +fun sget1(): slice { return str1; } +fun sget2(): slice { return str2; } +fun sget2r(): slice { return str2r; } + +const int240: int = ((int1+int2)*10)<<3; + +fun iget240(): MInt { return int240; } + +@pure +fun newc(): builder +asm "NEWC"; +@pure +fun builder.endcs(self): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): MInt +asm "SDEQ"; +@pure +fun builder.stslicer(self, s: slice): builder +asm "STSLICER"; + +@method_id(101) +fun test1() { + return (strange_zero, strange_minus_1); +} + +@method_id(102) +fun test2() { + return (true1, true2, true3); +} + +@method_id(103) +fun test3() { + return (false1, false2); +} + +@method_id(104) +fun test4() { + __expect_type(tens1, "(int, int)"); + return (tens1.0, tens2.2); +} + +@method_id(105) +fun test5() { + return (intOrN == null, int32Or64 is int32, int32Or64); +} + +@method_id(106) +fun test6() { + __expect_type(addr1, "address"); + return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain()); +} + +fun main() { + var i1: int = iget1(); + var i2: int = iget2(); + var i3: int = iget3(); + + assert(i1 == 1) throw int101; + assert(i2 == 2) throw 102; + assert(i3 == 3) throw 103; + + var s1: slice = sget1(); + var s2: slice = sget2(); + var s3: slice = newc().stslicer(str1).stslicer(str2r).endcs(); + + assert(sdeq(s1, newc().storeUint(str1int, 12 * nibbles).endcs())) throw int111; + assert(sdeq(s2, newc().storeUint(str2int, 6 * nibbles).endcs())) throw 112; + assert(sdeq(s3, newc().storeUint(0x636f6e737431AABBCC, 18 * nibbles).endcs())) throw 113; + + var i4: int = iget240(); + assert(i4 == 240) throw ((104)); + return 0; +} + +/** +@testcase | 0 | | 0 +@testcase | 101 | | 0 -1 +@testcase | 102 | | -1 -1 -1 +@testcase | 103 | | 0 0 +@testcase | 104 | | 1 1 2 +@testcase | 105 | | -1 0 7 48 +@testcase | 106 | | -1 0 0 -1 + +@code_hash 32362412747322136329528616455651783746542516198110452861733590440068294458753 +*/ diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 606318cb3..f53debb7f 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-dicts" -fun addIntToIDict(mutate self: cell?, key: int, number: int): void { +fun cell?.addIntToIDict(mutate self, key: int, number: int): void { return self.iDictSetBuilder(32, key, beginCell().storeInt(number, 32)); } @@ -14,7 +14,7 @@ fun calculateDictLen(d: cell?) { return len; } -fun loadTwoDigitNumberFromSlice(mutate self: slice): int { +fun slice.loadTwoDigitNumberFromSlice(mutate self): int { var n1 = self.loadInt(8); var n2 = self.loadInt(8); return (n1 - 48) * 10 + (n2 - 48); @@ -47,7 +47,7 @@ fun test102() { while (!shouldBreak) { var (kDel, kVal, wasDel) = dict.iDictDeleteLastAndGet(32); if (wasDel) { - deleted.tuplePush([kDel, kVal!.loadInt(32)]); + deleted.push([kDel, kVal!.loadInt(32)]); } else { shouldBreak = true; } @@ -82,8 +82,8 @@ fun test104() { var (old2, _) = dict.sDictDeleteAndGet(32, "key1"); var (restK, restV, _) = dict.sDictGetFirst(32); var (restK1, restV1, _) = dict.sDictDeleteLastAndGet(32); - assert (restK!.isSliceBitsEqual(restK1!)) throw 123; - assert (restV!.isSliceBitsEqual(restV1!)) throw 123; + assert (restK!.bitsEqual(restK1!)) throw 123; + assert (restV!.bitsEqual(restV1!)) throw 123; return ( old1!.loadTwoDigitNumberFromSlice(), old2!.loadTwoDigitNumberFromSlice(), @@ -93,6 +93,17 @@ fun test104() { ); } +@method_id(105) +fun test105(takeNext: bool) { + var dict = createEmptyDict(); + dict.uDictSet(32, 0, stringHexToSlice("01")); + dict.uDictSet(32, 8, stringHexToSlice("02")); + var (next, data, found) = takeNext ? dict.uDictGetNext(32, -1) : dict.uDictGetPrev(32, 9); + __expect_type(found, "bool"); + assert(found, 10); + return data!.loadUint(8); +} + fun main() {} /** @@ -102,4 +113,6 @@ fun main() {} @testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] @testcase | 103 | | 1 1 2 1 3 (null) @testcase | 104 | | 12 34 56 78 0 +@testcase | 105 | -1 | 1 +@testcase | 105 | 0 | 2 */ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index ca3109275..b92f27a6e 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -1,8 +1,17 @@ +type MInt = int; +type Tensor2Int = (int, int); +type Tensor2IntN = (int, int)?; + +@noinline fun eq1(value: X): X { return value; } fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } fun eq4(value: X) { return eq1(value); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var (a, b, c) = (x, (x,x), [x,x]); @@ -13,32 +22,44 @@ fun getTwo(): X { return 2 as X; } fun takeInt(a: int) { return a; } +struct Wrapper { + value: T; +} +struct Err { + errPayload: T +} + @method_id(102) -fun test102(): (int, int, int, [int, int]) { +fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { var a: int = getTwo(); var _: int = getTwo(); var b = getTwo() as int; var c: int = 1 ? getTwo() : getTwo(); var c redef = getTwo(); + var d: int = match (1 as int|slice|builder) { int => getTwo(), slice => getTwo(), builder => getTwo() }; var ab_tens = (0, (1, 2)); ab_tens.0 = getTwo(); ab_tens.1.1 = getTwo(); var ab_tup = [0, [1, 2]]; ab_tup.0 = getTwo(); ab_tup.1.1 = getTwo(); - return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); + var cint: Wrapper = { value: getTwo() }; + var cuni: Wrapper | Err = Wrapper { value: getTwo() }; + cuni = Err { errPayload: getTwo() }; + return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1], eq1({ value: { value: getTwo() } } as Wrapper< Wrapper >)); } @method_id(103) fun test103(first: int): (int, int, int) { var t = createEmptyTuple(); var cs = beginCell().storeInt(100, 32).endCell().beginParse(); - t.tuplePush(first); - t.tuplePush(2); - t.tuplePush(cs); - cs = t.tupleAt(2); - cs = t.tupleAt(2) as slice; - return (t.tupleAt(0), cs.loadInt(32), t.tupleAt(2).loadInt(32)); + t.push(first); + t.push(2); + t.push(cs); + cs = t.get(2); + cs = t.get(2) as slice; + return (t.get(0), cs.loadInt(32), t.get(2).loadInt(32)); } fun manyEq(a: T1, b: T2, c: T3): [T1, T2, T3] { @@ -85,19 +106,21 @@ fun test106() { ]; } -fun callTupleFirst(t: X): Y { return t.tupleFirst(); } -fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } -fun getTupleLastInt(t: tuple) { return t.tupleLast(); } -fun getTupleSize(t: tuple) { return t.tupleSize(); } +fun X.callTupleFirst(self): Y { return self.first(); } +fun callTupleFirst(t: X): Y { return t.first(); } +fun T.callTuplePush(mutate self, v1: V, v2: V): self { self.push(v1); tuplePush(mutate self, v2); return self; } +fun getTupleLastInt(t: tuple) { return t.last(); } +fun getTupleSize(t: MTuple) { return t.size(); } fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } -global t107: tuple; +global t107: MTuple; +type MTuple = tuple; @method_id(107) fun test107() { t107 = createEmptyTuple(); - callTuplePush(mutate t107, 1, 2); + t107.callTuplePush(1, 2); t107.callTuplePush(3, 4).callTuplePush(5, 6); var first: int = t107.callTupleFirst(); return ( @@ -105,7 +128,7 @@ fun test107() { callAnyFn2(getTupleSize, t107), first, callTupleFirst(t107) as int, - callAnyFn(getTupleLastInt, t107), + callAnyFn(getTupleLastInt, t107 as tuple), callAnyFn2(getTupleLastInt, t107) ); } @@ -131,7 +154,64 @@ fun test108() { return g108; } -fun main(x: int): (int, [[int, int]]) { +@method_id(109) +fun test109(initialX: int32) { + var x: MInt = initialX; + __expect_type(eq1(x), "MInt"); + __expect_type(eq1(x), "int"); + __expect_type(eq1(x), "int8?"); + return (eq1(x), eq1(null), eq1(null)); +} + +@method_id(110) +fun test110() { + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); + ab_tens_al.0 = getTwo(); + var ab_tens_al_n: Tensor2Int? = (5, 6); + ab_tens_al_n.1 = getTwo(); + return (ab_tens_al, ab_tens_al_n); +} + +type Tup2Int = [int, int]; + +fun isSomeNullableNull(v: T?) { return v == null; } + +type MIntN = int?; +type Tup2IntN = Tup2Int?; + +@method_id(111) +fun test111() { + var i1: MIntN = 3; + var i2: Tup2IntN = [1, 2]; + return (isSomeNullableNull(i1), isSomeNullableNull(i2), isSomeNullableNull(i1), isSomeNullableNull(i1)); +} + +@method_id(112) +fun test112(v: int | slice?) { + return (eq1(v), eq4(v == null)); +} + +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +@method_id(113) +fun test113(a: int | slice) { + var b: int | (int, int) = match (a) { + int => a, + slice => (a.loadInt(32), -1) + }; + __expect_type(makeNullable(b), "int | (int, int) | null"); + return ( + makeNullable(a), -100, makeNullable(b), -100, makeNullable(9), -100, makeNullable(null), -100, + makeNullable(b), -100, makeNullable(null) + ); +} + + +fun main(x: int): (int, [Tup2Int]) { + __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); + try { if(x) { throw (1, x); } } catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } return (0, [[x, 1]]); @@ -140,21 +220,33 @@ fun main(x: int): (int, [[int, int]]) { /** @testcase | 0 | 1 | 1 [ [ 1 2 ] ] @testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] -@testcase | 102 | | 2 2 2 [ 2 2 ] +@testcase | 102 | | 2 2 2 [ 2 2 ] 2 @testcase | 103 | 0 | 0 100 100 @testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] @testcase | 105 | | 3 @testcase | 106 | | [ 106 2 6 6 ] @testcase | 107 | | 6 6 1 1 6 6 @testcase | 108 | | 45 - -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1<(int,int)> -@fif_codegen DECLPROC eq1<[int,int]> -@fif_codegen DECLPROC getTwo - -@fif_codegen_avoid DECLPROC eq1 -@fif_codegen_avoid DECLPROC eq2 -@fif_codegen_avoid DECLPROC eq3 +@testcase | 109 | 5 | 5 (null) (null) +@testcase | 110 | | 2 2 5 2 +@testcase | 111 | | 0 0 0 0 +@testcase | 112 | 5 1 | 5 1 0 +@testcase | 112 | 0 0 | 0 0 -1 +@testcase | 113 | 5 1 | 5 1 -100 (null) 5 1 -100 9 -100 (null) -100 (null) 5 1 -100 (null) (null) (null) 0 + +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1<(int,int)>() +@fif_codegen DECLPROC eq1<[int,int]>() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1>>() + +// was inlined +@fif_codegen_avoid DECLPROC getTwo() +@fif_codegen_avoid getTwo + +@fif_codegen_avoid DECLPROC eq1() +@fif_codegen_avoid DECLPROC eq2() +@fif_codegen_avoid DECLPROC eq3() */ diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk new file mode 100644 index 000000000..2a6684f0d --- /dev/null +++ b/tolk-tester/tests/generics-2.tolk @@ -0,0 +1,486 @@ +struct Empty {} + +struct Wrapper { + value: T; +} + +type PairAlias = Pair; +struct Pair { + first: T1, + second: T2 +} + +type MyInt = int; +type WrappedInt = Wrapper; +type WrapperAlias = Wrapper; + +struct DeepNested { + wrapper: WrapperAlias>; + pair: PairAlias; +} + + +fun eq(v: T): T { return v; } + +fun swap(p: Pair): Pair { + return { first: p.second, second: p.first }; +} + +fun wrap(value: T): Wrapper { + return { value }; +} + +fun wrap2(value: T) { + return WrapperAlias { value }; +} + +fun makePair(a: A, b: B) { + var c: Pair = { first: a, second: b }; + return c; +} + + +@method_id(101) +fun test1() { + var c1: Wrapper = { value: 23 }; + __expect_type(c1, "Wrapper"); + var c2: WrapperAlias = { value: 23 }; + __expect_type(c2, "WrapperAlias"); + __expect_type(c2 as Wrapper, "Wrapper"); + __expect_type(c2 as WrappedInt, "WrappedInt"); + return (c1, c2); +} + +@method_id(102) +fun test2(c: Wrapper) { + var c2 = Wrapper { value: c.value + 1 }; + var c3 = Wrapper { value: c.value + c2.value }; + __expect_type(c3, "Wrapper"); + var c4: Wrapper? = c.value > 10 ? null : { value: c.value }; + return (c, c2, c3, c4); +} + +@pure +@noinline +fun getWrappervalue1(c: Wrapper) { + return c.value; +} + +@pure +fun Wrapper.getWrappervalue1(self) { + return self.value; +} + +@pure +@noinline +fun getWrappervalue2(c: T) { + return c.value; +} + +@pure +fun T.getWrappervalue2(self) { + return self.value; +} + +@method_id(103) +fun test3() { + var c1 = wrap(10); + var c2 = wrap(10 as int?); + var c3 = wrap2<(int, int)?>(null); + __expect_type(c1, "Wrapper"); + __expect_type(c2, "Wrapper"); + __expect_type(c3, "Wrapper<(int, int)?>"); + return (getWrappervalue1(c1), getWrappervalue1(c2), getWrappervalue1(c3), 777, getWrappervalue2(c1), getWrappervalue2(c2), getWrappervalue2(c3)); +} + +@method_id(104) +fun test4() { + var c1 = wrap(10); + var c2 = wrap(c1); + var c3 = wrap(Wrapper { value: beginCell() }); + __expect_type(c2, "Wrapper>"); + __expect_type(c3, "Wrapper>"); + __expect_type(c3.value, "Wrapper"); + __expect_type(c3.value.value, "builder"); + c3.value.value.storeInt(1, 32).storeInt(2, 32).storeInt(3, 32); + var c4 = wrap2(Wrapper { value: getWrappervalue1(c3.value).endCell().beginParse() }); + __expect_type(c4, "Wrapper>"); + var sumOf1 = 0; + sumOf1 += getWrappervalue1(getWrappervalue1(c4)).loadInt(32); // a function returns a copy + sumOf1 += getWrappervalue2(c4.value).loadInt(32); // so original slice isn't mutated + return (c1.value += c4.value.value.loadInt(32), c1.getWrappervalue1(), c1.value += c4.value.value.loadInt(32), c1.getWrappervalue2(), c1, sumOf1); +} + +@method_id(105) +fun test5(setNull: bool) { + var c1: Wrapper < WrappedInt >? = { value: { value: 80 } }; + __expect_type(c1, "Wrapper"); + if (setNull) { + c1 = null; + } + __expect_type(c1, "Wrapper?"); + return c1; +} + +fun createNestedWrapperOrNull(setNull: bool, initVal: T) { + if (setNull) { + return null; + } + var c1 = Wrapper> {value:{value:initVal}}; + var c2: Wrapper< Wrapper<(T)> > = {value:{value:initVal}}; + var c3: Wrapper< Wrapper<((T | T))> > = WrapperAlias {value:{value:initVal}}; + var c4: Wrapper< Wrapper<((T | T))> > = Wrapper> {value:c3.value}; + var c5: Wrapper< WrapperAlias<((T | T))> > = WrapperAlias> {value:c3.value}; + var c6: WrapperAlias< WrapperAlias<((T | T))> > = Wrapper> {value:c3.value}; + return c1; +} + +@method_id(106) +fun test6() { + var c1 = createNestedWrapperOrNull(true, (1, 2)); + var c2 = createNestedWrapperOrNull(false, (1, 2)); + __expect_type(c1, "Wrapper>?"); + getWrappervalue1(c2!); + getWrappervalue2(c2!); + return (c1, 777, c2); +} + +@method_id(107) +fun test7(i8: int8) { + var c1 = Pair { first: 10, second: i8 }; + var c2 = eq(c1); + __expect_type(c2, "Pair"); + var c3: Pair = { first: null, second: null }; + return (c2.first += 10, c2, swap(c2), 777, c3); +} + +@method_id(108) +fun test8(lastNull: bool) { + var c1 = Wrapper { value: () }; + var c2 = Pair> { first: {}, second: {first:{},second:{}} }; + var c3 = Pair { first: c1, second: Wrapper { value: Empty{} } }; + var c4 = Pair { first: c1, second: Wrapper { value: lastNull ? null : Empty{} } }; + __expect_type(c4, "Pair, Wrapper>"); + return (c1, c2, c3, 777, c4, 777, c4.first, c4.second.value); +} + +@method_id(109) +fun test9() { + var c1: WrappedInt = { value: 10 }; + var c2: WrappedInt = WrappedInt { value: 20 }; + var c3: WrappedInt = Wrapper { value: 30 }; + var c4: Wrapper = { value: 40 }; + c1 = c2 = c3 = c4; + __expect_type(c1, "WrappedInt"); + __expect_type(c1 as Wrapper, "Wrapper"); + + var c5: Pair, WrappedInt> = { first: {value:50}, second: {value:50} }; + c5.first = c1; c5.first = c4; + c5.second = c1; c5.second = c4; + + var c6 = Wrapper> { value: { value: 60 } }; + var c7 = Wrapper { value: { value: 70 } }; + c6 = c7; c7 = c6; + c6 as Wrapper; + c7 as Wrapper>; + + var p1: PairAlias = { first: 42, second: WrappedInt { value: 200 } }; + __expect_type(p1, "PairAlias"); + __expect_type(p1 as Pair>, "Pair>"); + var p2: Pair> = p1; + + __expect_type(DeepNested { + wrapper: wrap(wrap(999)), + pair: makePair(10, 20), + }, "DeepNested"); + + return (c5, c7); +} + +type Int16Or8 = int8 | int16; + +@method_id(110) +fun test10() { + var p1 = Pair { first: 5, second: 10 as int8 }; + var p2: Pair = { first: 10, second: 5 as int16 }; + var p3 = Pair { first: 20 as int|builder, second: (20 as int8) as Int16Or8 }; + __expect_type(p1, "Pair"); + __expect_type(p2, "Pair"); + __expect_type(p3, "Pair"); + p1 = p2; p1 = p3; p2 = p1; p2 = p3; p3 = p1; p3 = p2; + p2 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + return p3; +} + +@method_id(111) +fun test11() { + // type_id of these instantiations are the same + var c1 = { value: 5 } as int | Wrapper; + var c2 = { value: 5 } as int | Wrapper; + var c3 = { value: 5 } as int | Wrapper; + var c4 = { value: 5 } as int | Wrapper; + c1 = c1; c1 = c2; c1 = c3; c1 = c4; + c2 = c1; c2 = c2; c2 = c3; c2 = c4; + c3 = c1; c3 = c2; c3 = c3; c3 = c4; + c4 = c1; c4 = c2; c4 = c3; c4 = c4; + return (c1, c2, c3, c4); +} + +@method_id(112) +fun test12(is1: bool) { + var c1: Wrapper = { value: 10 }; + var c2: Wrapper = { value: 20 }; + var c3: Wrapper | WrapperAlias = is1 ? c1 : c2; + __expect_type(c3, "Wrapper"); + var m3: Wrapper | Wrapper = is1 ? c1 : Wrapper { value: null }; + __expect_type(m3, "Wrapper | Wrapper"); + return (c3, 777, m3); +} + +fun makeIdentity(): (T) -> T { + return eq; +} + +@method_id(113) +fun test13() { + return makeIdentity>()({ first: 30, second: true }); +} + +struct Storage { + Wrapper: Wrapper; + backup: T?; +} + +@method_id(114) +fun test14() { + var s1 = Storage { Wrapper: wrap(999), backup: null }; + var s2 = Storage { Wrapper: wrap(makePair(1, "")), backup: makePair(2, "") }; + __expect_type(s1, "Storage"); + __expect_type(s2, "Storage>"); + return (s1, s2.backup!.first); +} + +struct DualStorage { + primary: Wrapper; + secondary: Wrapper; +} + +type IntOrBuilder = int | builder; + +@method_id(115) +fun test15() { + var s1 = DualStorage { primary: wrap(10), secondary: wrap(false), }; + __expect_type(s1, "DualStorage"); + var s2: DualStorage = { primary: wrap(100 as IntOrBuilder), secondary: wrap(true), }; + var s3 = DualStorage { primary: wrap(200 as IntOrBuilder), secondary: wrap(true), }; + return (s1, s2 = s3, s3 = s2); +} + +type ComplexWrapPair = Wrapper>; + +@method_id(116) +fun test16() { + var c1: ComplexWrapPair = wrap(makePair(1, 2)); + __expect_type(c1, "ComplexWrapPair"); + var c2 = wrap(makePair(3, 4)); + __expect_type(c2, "Wrapper>"); + if (0) { c1 = c2; c2 = c1; } + c1 as Wrapper>; c2 as ComplexWrapPair; + return (c1, c2); +} + +fun transform(x: T, f: (T) -> U): U { + return f(x); +} + +@method_id(117) +fun test17() { + var id = makeIdentity(); + var result = transform(100 as MyInt, id); + __expect_type(result, "MyInt"); + var r2 = transform(wrap(123), wrap>); + __expect_type(r2, "Wrapper>"); + return (result, r2); +} + +struct UnionHolder { + item: Wrapper; + extra: Wrapper; +} + +@method_id(118) +fun test18() { + var uh: UnionHolder = { item: wrap(123 as int|slice), extra: wrap("" as int|slice|bool) }; + __expect_type(uh, "UnionHolder"); + __expect_type(uh.item, "Wrapper"); + __expect_type(uh.extra, "Wrapper"); + + var ub = UnionHolder { item: wrap(123 as int|bool), extra: wrap(true as int|bool) }; + __expect_type(ub.extra, "Wrapper"); + + return (uh.item, 777, ub); +} + +type StrangeInt = int; +type StrangeAlsoInt = StrangeInt<()>; + +@method_id(119) +fun test19() { + var i1: StrangeInt = 20; + var i2: StrangeInt = 30; + var i3: StrangeInt> = 40 as StrangeAlsoInt; + i1 = i2; i2 = i1; i1 = i3; i3 = i1; + return (i1 as StrangeInt<()>, i3 as StrangeAlsoInt, i1 is StrangeInt, i1 is StrangeInt, i3 is StrangeInt, i3 is StrangeAlsoInt); +} + +@method_id(120) +fun test20() { + var w1: Wrapper | int = { value: 10 }; + var w2: int | Wrapper = { value: 30 }; + var w3: slice | Wrapper<(int8, int16)> = { value: (50, 30) }; + __expect_type(w3.value, "(int8, int16)"); + var w4 = { value: null } as int | Wrapper; + var w5 = { value: null } as int | Wrapper?>; + var w6: int | Wrapper | null | slice | builder | (int, int) = { value: 10 }; + if (10 > 3) { + w6 = beginCell(); + } + __expect_type(w6, "builder | Wrapper"); + return (w4, 777, w5, w5 is Wrapper, w6 is builder); +} + +struct AnotherWrapper { + value: int; +} + +fun constructAnotherWrapper(value: slice): AnotherWrapper { + return AnotherWrapper { value }; +} + +fun test21(w: AnotherWrapper) { + var w2: AnotherWrapper = { value: beginCell() }; + var w3 = constructAnotherWrapper(beginCell()); + __expect_type(w2, "AnotherWrapper"); + __expect_type(w3, "AnotherWrapper"); + __expect_type(constructAnotherWrapper>, "(Wrapper) -> AnotherWrapper>"); + w.value = w2.value; + w3.value = beginCell(); + w = w2; +} + +fun test22(w: int | WrapperAlias | slice) { + match (w) { + int => {} + WrapperAlias => {} + slice => {} + } + __expect_type(w, "int | Wrapper | slice"); + match (w) { + MyInt => {} + slice => {} + WrapperAlias => {} + } +}; + +@method_id(123) +fun test23() { + var w1: Wrapper = { value: 10 }; + var w2: Wrapper> = { value: w1 }; + var w3: Wrapper>> = { value: w2 }; + var w4: Wrapper>>> = { value: w3 }; + var w5: Wrapper>>>> = { value: w4 }; + eq>(w1); + eq>>(w2); + eq>>>(w3); + return (w1, w2, w3, w4, w5); +} + +fun takeSomethingAndWrapper(something: T, w: Wrapper) { + w.value = something; +} + +struct HasSomethingAndWrapper { + something: T; + w: Wrapper; +} + +fun test24() { + // check that parameter w can be understood from context (after deducing parameter something) + takeSomethingAndWrapper(0, {value: 0}); + takeSomethingAndWrapper(null as slice?, {value: null}); + takeSomethingAndWrapper(Wrapper { value: 10 }, {value: { value: 20 }}); + // same for creating an object and deducing Ts one by one + HasSomethingAndWrapper { something: 0, w: { value: 0 } }; + val o = HasSomethingAndWrapper { something: 5 as uint8, w: { value: 5 } }; + __expect_type(o, "HasSomethingAndWrapper"); +} + +fun test25() { + var p1 = Pair { first: null, second: null }; + __expect_type(p1, "Pair"); + + try {} + catch(e, second) { + var p2 = Pair { first: null, second }; + __expect_type(p2, "Pair"); + } +} + + +fun main(c: Wrapper, d: WrappedInt) { + __expect_type(c, "Wrapper"); + __expect_type(c!, "Wrapper"); + __expect_type(d, "WrappedInt"); + __expect_type(d.value, "int"); + __expect_type(WrappedInt { value: 200 }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value as bytes32 }, "Wrapper"); + __expect_type(Wrapper{ value: Wrapper { value: (c.value as bytes32?)! }.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value.loadInt(32) }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 } as WrapperAlias, "WrapperAlias"); + __expect_type(swap(Pair { first: 3, second: beginCell() }), "Pair"); +} + +/** +@testcase | 101 | | 23 23 +@testcase | 102 | 9 | 9 10 19 9 +@testcase | 102 | 19 | 19 20 39 (null) +@testcase | 103 | | 10 10 (null) (null) 0 777 10 10 (null) (null) 0 +@testcase | 104 | | 11 11 13 13 13 2 +@testcase | 105 | 0 | 80 +@testcase | 105 | -1 | (null) +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-14 +@testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 +@testcase | 108 | 0 | 777 typeid-15 777 typeid-15 +@testcase | 108 | -1 | 777 0 777 0 +@testcase | 109 | | 40 40 70 +@testcase | 110 | | 20 1 20 42 +@testcase | 111 | | 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 +@testcase | 112 | -1 | 10 1 777 10 1 typeid-5 +@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-6 +@testcase | 113 | | 30 -1 +@testcase | 114 | | 999 (null) 2 +@testcase | 115 | | 10 0 200 1 -1 200 1 -1 +@testcase | 116 | | 1 2 3 4 +@testcase | 117 | | 100 123 +@testcase | 118 | | 123 1 777 123 1 -1 2 +@testcase | 119 | | 40 40 -1 -1 -1 -1 +@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-11 -1 -1 +@testcase | 123 | | 10 10 10 10 10 + + +@fif_codegen +""" + test1() PROC:<{ + 23 PUSHINT // c1=23 + DUP // c1=23 c2=23 + }> +""" + +@fif_codegen DECLPROC getWrappervalue1>() +@fif_codegen DECLPROC getWrappervalue2>>() + */ diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk new file mode 100644 index 000000000..494e48ba2 --- /dev/null +++ b/tolk-tester/tests/generics-3.tolk @@ -0,0 +1,145 @@ +struct Ok { + result: T; +} + +struct Err { + errPayload: T; +} + +type OkAlias = Ok; + +type Response = Ok | Err; + +@pure +fun getResponse(success: bool): Response { + return success ? Ok { result: 10 } : Err { errPayload: beginCell().storeInt(-1,32).endCell().beginParse() }; +} + +@method_id(101) +fun test1(getOk: bool) { + return match (var r = getResponse(getOk)) { + Ok => r.result, + Err => r.errPayload.loadInt(32) + }; +} + +fun test2() { + var r = getResponse(true); + match (r) { + Ok => r as OkAlias, + Err => r as Err, + } + match (r) { + OkAlias => r as Ok, + Err => {} + } + match (r) { + Ok => { __expect_type(r, "Ok"); } + Err => { __expect_type(r, "Err"); } + } + + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is OkAlias) { __expect_type(r, "Ok"); } + if (r !is OkAlias) { __expect_type(r, "Err"); } + if (r is Ok && r !is Err && r is Ok) { __expect_type(r, "Ok"); } +} + +@method_id(103) +fun test3() { + var r: Response = getResponse(true); + r = Err { errPayload: "" }; + __expect_type(r, "Err"); + r = OkAlias { result: 10 }; + __expect_type(r, "Ok"); + return (r, r is Ok, (r as Response) is Err); +} + +@method_id(104) +fun test4() { + var r1 = OkAlias { result: OkAlias { result: 10 } } as Response< Ok, slice >; + match (r1) { + Ok => { __expect_type(r1, "Ok>"); r1.result.result; } + Err => {} + } + var r2: Response< Response, slice > = Err { errPayload: "" }; + __expect_type(r2, "Err"); + r2 = 10>3 ? Ok { result: Ok { result: 12 } } : r2; + var mm1 = match (r2) { + OkAlias> => true, + Err => false + }; + var mm2 = match (r2) { + Ok => true, + Err => false + }; + return (r2, mm1, mm2, r2 is Ok>, r2 is OkAlias>, r2 is Err, r2 is Ok, r2 is Err, r2 !is Ok); +} + +@method_id(105) +fun test5() { + var o1: Ok = { result: 1 }; + var o2: OkAlias = { result: 2 }; + return (o1 is Ok, o1 is OkAlias, o2 is Ok, o2 is OkAlias, o1 is Ok, o1 is OkAlias); +} + +type OkInt = Ok; + +fun test6(w: Response | int) { + match (w) { + int => {} + Ok => {} + Err => {} + } + match (w) { + OkInt => {} + int => {} + Err => {} + } +} + +fun test7(w: OkInt | Err | int) { + match (w) { + Ok => {} + Err => {} + int => {} + } +} + +struct WithDef { + f1: T? = null; + f2: int? = null as int?; + f3: T? = null as T?; + price: coins = ton("0.05"), + slice: slice = stringHexToSlice("010203"), +} + +@method_id(108) +fun test8() { + var w1: WithDef = {}; + var w2: WithDef> = { price: ton("0.1"), f3: { result: 12 } }; + return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.remainingBitsCount()); +} + + +fun main() { + __expect_type(getResponse(true), "Response"); +} + +/** +@testcase | 101 | -1 | 10 +@testcase | 101 | 0 | -1 +@testcase | 103 | | 10 -1 0 +@testcase | 104 | | 12 typeid-1 typeid-4 -1 -1 -1 0 0 -1 0 0 +@testcase | 105 | | -1 -1 -1 -1 0 0 +@testcase | 108 | | 50000000 (null) (null) 100000000 12 12 24 + +@fif_codegen +""" + test3() PROC:<{ + 10 PUSHINT // r.USlot1=10 + -1 PUSHINT // r.USlot1=10 '22=-1 + FALSE // r.USlot1=10 '22=-1 '24 + }> +""" + */ diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk new file mode 100644 index 000000000..c5f3e038c --- /dev/null +++ b/tolk-tester/tests/generics-4.tolk @@ -0,0 +1,205 @@ +@noinline +fun eqUnusedT(v: int) { return v; } +@noinline +fun eqUnusedU(v: T) { if (v is int123) { __expect_type(v as U, "never"); } return v; } + +fun dup(x: T1, y: T2): (T1, T2) { return (x, y) } + +struct Container1 { + item: T; +} + +fun getItemOf(v: Container1) { + return v.item; +} + +struct WithDef1 { + body: T? = null; +} + +fun getBodyOf(o: WithDef1) { + return o.body; +} + +struct WithNever { + f1: int; + f2: T; +} + +fun eqNever(o: WithNever) { return o; } + +struct MyInit { + value: coins; + data: TInit; +} + +fun MyInit.getValue(self) { + return self.value; +} + +struct Parameters { + bounce: bool; + body: TBody; + init: builder | MyInit | null = null; +} + +fun createParameters(bounce: bool, body: TBody, data: TInit): Parameters { + return { bounce, body, init: { value: ton("0"), data } }; +} + +@noinline +fun mySend(p: Parameters): int { + var total = 0; + if (p.bounce) { + total += 1; + } + if (p.body !is never) { + assert(p.body is TBody, 101); + total += 10; + } + if (p.init is MyInit) { + total += 100 + p.init.getValue(); + } + return total; +} + +fun test1() { + eqUnusedT(100); + eqUnusedU(100); + eqUnusedU(beginCell()); + + __expect_type(dup(null, null), "(null, null)"); + __expect_type(dup(createEmptyTuple(), 6), "(tuple, int)"); +} + +fun test2() { + var w1 = Container1 { item: 123 }; + var w2 = Container1 { item: null }; + __expect_type(w1, "Container1"); + __expect_type(w2, "Container1"); + __expect_type(getItemOf(w1), "int"); + __expect_type(getItemOf(w2), "null"); + + __expect_type(getItemOf({item: null as slice?}), "slice?"); + __expect_type(getItemOf({item: null}), "null"); +} + +@method_id(103) +fun test3() { + __expect_type(WithNever{f1:10}, "WithNever"); + __expect_type(WithNever{f1:10,f2:20}, "WithNever"); + + __expect_type(eqNever({f1:10}), "WithNever"); + __expect_type(eqNever({f1:10,f2:20}), "WithNever"); + __expect_type(eqNever({f1:10,f2:null}), "WithNever"); + + var a: WithNever = {f1:10}; + return (a, WithNever{f1:20}, eqNever({f1:30}), 777, eqNever({f1:40,f2:40})); +} + +@method_id(104) +fun test4() { + __expect_type(getBodyOf({body: 123}), "int?"); + __expect_type(getBodyOf({body: null}), "null"); + __expect_type(getBodyOf({}), "null"); + + __expect_type(WithDef1{}, "WithDef1"); + __expect_type(WithDef1{}.body, "null"); + __expect_type(WithDef1{body: null}.body, "null"); + __expect_type(WithDef1{body: 123}.body, "int?"); + + return (getBodyOf({body: null}), getBodyOf({}), WithDef1{}); +} + +@method_id(105) +fun test5() { + __expect_type(Parameters { bounce: true }, "Parameters"); + __expect_type(Parameters { bounce: false, body: 179 }, "Parameters"); + __expect_type(Parameters { bounce: true, init: beginCell() }, "Parameters"); + __expect_type(Parameters { bounce: false, body: beginCell(), init: { value: 123, data: 123 } }, "Parameters"); + + __expect_type(createParameters(true, null, null), "Parameters"); + __expect_type(createParameters(true, beginCell(), "123"), "Parameters"); + + __expect_type(createParameters(true, null, null).body, "null"); + __expect_type(createParameters(true, 123, null).body, "int"); + __expect_type(createParameters(true, null, null).init, "builder | MyInit | null"); + __expect_type(createParameters(true, 123, 456).init, "builder | MyInit | null"); + + return (createParameters(true, null, null), 777, createParameters(false, 123, 456)); +} + +@method_id(106) +fun test6() { + var p: Parameters = { + bounce: true, + body: 123, + init: { value: ton("0"), data: beginCell().endCell() } + }; + return (p.body is int, p.init is cell, p.init is MyInit, p.init is MyInit && p.init.data.depth() == 0); +} + +@method_id(107) +fun test7() { + __expect_type(Parameters{ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}, "Parameters"); + var v1 = mySend({ bounce: true }); + var v2 = mySend({ bounce: false, body: 123, init: beginCell() }); + var v3 = mySend({ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}); + var v4 = mySend({ bounce: true, init: { value: 16, data: null as [int]? }}); + return (v1, v2, v3, v4); +} + +struct FakeGeneric8 { + alwaysInt: int; +} + +struct Snake8 { + next: FakeGeneric8; // it's not a recursive struct, it's okay + next2: FakeGeneric8?; + next3: FakeGeneric8>; +} + +@method_id(108) +fun test8() { + var sn: Snake8 = { + next: { alwaysInt: 10 }, + next2: null, + next3: { alwaysInt: 20 } + }; + return sn; +} + +fun main() { +} + +/** +@testcase | 103 | | 10 20 30 777 40 40 +@testcase | 104 | | (null) (null) (null) +@testcase | 105 | | -1 (null) 0 (null) typeid-6 777 0 123 0 456 typeid-5 +@testcase | 106 | | -1 0 -1 -1 +@testcase | 107 | | 1 10 110 117 +@testcase | 108 | | 10 (null) 20 + +@fif_codegen DECLPROC eqUnusedT() +@fif_codegen DECLPROC eqUnusedU() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() + +@fif_codegen +""" + test6() PROC:<{ // + NEWC // '8 + ENDC // p.init.USlot2 + -1 PUSHINT // p.init.USlot2 '11=-1 + FALSE // p.init.USlot2 '11=-1 '12 + TRUE // p.init.USlot2 '11=-1 '12 '14 + s0 s3 XCHG // '14 '11=-1 '12 p.init.USlot2 + CDEPTH // '14 '11=-1 '12 '19 + 0 EQINT // '14 '11=-1 '12 '21 + 0 NEQINT // '14 '11=-1 '12 '18 + s1 s3 s0 XCHG3 // '11=-1 '12 '14 '18 + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-1.tolk b/tolk-tester/tests/handle-msg-1.tolk new file mode 100644 index 000000000..1e08082d5 --- /dev/null +++ b/tolk-tester/tests/handle-msg-1.tolk @@ -0,0 +1,134 @@ + +fun setTvmRegisterC7(c7: [tuple]): void + asm "c7 POP"; + +@noinline +fun emulateC7PresentForTests(bounced: bool, senderAddress: address, fwdFee: coins, createdLt: int, createdAt: int, valueCoins: coins, valueExtra: dict) { + var c7_inner = createEmptyTuple(); + repeat (17) { + c7_inner.push(null); + } + + var inmsgparams = createEmptyTuple(); + inmsgparams.push(null); // bounce (not present in InMessage) + inmsgparams.push(bounced); + inmsgparams.push(senderAddress); + inmsgparams.push(fwdFee); + inmsgparams.push(createdLt); + inmsgparams.push(createdAt); + inmsgparams.push(null); // orig value (not present in InMessage) + inmsgparams.push(valueCoins); // in onInternalMessage, from a stack; in onBouncedMessage, from TVM + inmsgparams.push(valueExtra); + inmsgparams.push(null); // state init (not present in InMessage) + c7_inner.push(inmsgparams); + + setTvmRegisterC7([c7_inner]); +} + +fun invokeTest(body: slice, method_id: int): void + asm "c3 PUSH" "EXECUTE"; // stack: body + +@method_id(101) +fun handle1(in: InMessage) { + return in.senderAddress.getWorkchain(); +} + +@method_id(201) +fun entrypoint1() { + emulateC7PresentForTests(true, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 900, null); + invokeTest("", 101); +} + +@method_id(102) +fun handle2(in: InMessage) { + return (in.valueCoins, in.senderAddress.isInternal()); +} + +@method_id(202) +fun entrypoint2() { + emulateC7PresentForTests(false, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 888, null); + invokeTest("", 102); +} + +@method_id(103) +fun handle3(`in()`: InMessage) { + return ( + `in()`.senderAddress.getWorkchainAndHash(), + `in()`.valueCoins, + `in()`.valueExtra, + `in()`.createdLt, + `in()`.createdAt, + `in()`.body.isEmpty(), + ) +} + +@method_id(203) +fun entrypoint3() { + emulateC7PresentForTests(true, address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"), 100, 20, 30, 999, null); + invokeTest("00", 103); +} + +fun onInternalMessage(in: InMessage) { + __expect_type(in.body, "slice"); + __expect_type(in.senderAddress, "address"); + __expect_type(in.valueCoins, "coins"); + return in.originalForwardFee; +} + +/** +@testcase | 201 | | 0 +@testcase | 202 | | 888 -1 +@testcase | 203 | | -1 65535 999 (null) 20 30 0 + +@fif_codegen +""" + handle1() PROC:<{ // in.body + DROP // + INMSG_SRC // '1 + REWRITESTDADDR + DROP // '3 + }> +""" + +@fif_codegen +""" + handle2() PROC:<{ // in.body + DROP // + INMSG_VALUE // '1 + INMSG_SRC // '1 '3 + b{10} SDBEGINSQ + NIP // '1 '5 + }> +""" + +@fif_codegen +""" + handle3() PROC:<{ + INMSG_SRC + REWRITESTDADDR + INMSG_VALUE + INMSG_VALUEEXTRA + INMSG_LT + INMSG_UTIME + s0 s6 XCHG + SEMPTY + s5 s6 XCHG + s4 s5 XCHG + s3 s4 XCHG + s1 s3 s0 XCHG3 + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-2.tolk b/tolk-tester/tests/handle-msg-2.tolk new file mode 100644 index 000000000..2f8a8b29f --- /dev/null +++ b/tolk-tester/tests/handle-msg-2.tolk @@ -0,0 +1,23 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-3.tolk b/tolk-tester/tests/handle-msg-3.tolk new file mode 100644 index 000000000..41d3c1734 --- /dev/null +++ b/tolk-tester/tests/handle-msg-3.tolk @@ -0,0 +1,31 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + in.bouncedBody.skipBouncedPrefix(); + throw in.bouncedBody.loadUint(32); +} + +fun onInternalMessage(in: InMessage) { + return in.body; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + 32 LDU + NIP + 32 PLDU + THROWANY + }> // in.body + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-4.tolk b/tolk-tester/tests/handle-msg-4.tolk new file mode 100644 index 000000000..ac772992d --- /dev/null +++ b/tolk-tester/tests/handle-msg-4.tolk @@ -0,0 +1,40 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@inline_ref +fun onBouncedMessage(in: InMessageBounced) { + contract.setData(createEmptyCell()); +} + +fun onInternalMessage(in: InMessage) { + return 123; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROCREF:<{ // in.body + DROP + PUSHREF + c4 POP + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() INLINECALLDICT + }> + DROP + 123 PUSHINT + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-5.tolk b/tolk-tester/tests/handle-msg-5.tolk new file mode 100644 index 000000000..d5a30f8f9 --- /dev/null +++ b/tolk-tester/tests/handle-msg-5.tolk @@ -0,0 +1,39 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@noinline +fun onBouncedMessage(in: InMessageBounced) { + throw in.valueCoins; +} + +fun onInternalMessage(in: InMessage) { + return in.valueCoins; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() CALLDICT // + }> // in.body + DROP // + INMSG_VALUE // '3 + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-6.tolk b/tolk-tester/tests/handle-msg-6.tolk new file mode 100644 index 000000000..0d349f9cc --- /dev/null +++ b/tolk-tester/tests/handle-msg-6.tolk @@ -0,0 +1,47 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + if (in.valueExtra != null) { + return; + } + throw in.originalForwardFee; +} + +fun onInternalMessage(in: InMessage) { + return in.senderAddress; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + ISNULL + IFNOTJMP:<{ + }> + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + onBouncedMessage() CALLDICT + }> + DROP + INMSG_SRC + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-7.tolk b/tolk-tester/tests/handle-msg-7.tolk new file mode 100644 index 000000000..8410d6f3d --- /dev/null +++ b/tolk-tester/tests/handle-msg-7.tolk @@ -0,0 +1,22 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@on_bounced_policy("manual") +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk new file mode 100644 index 000000000..d510dd4a4 --- /dev/null +++ b/tolk-tester/tests/if-else-tests.tolk @@ -0,0 +1,429 @@ +global t: tuple; + +@method_id(101) +fun test1(x: int): int { + if (x > 200) { + return 200; + } else if (x > 100) { + return 100; + } else if (!(x <= 50)) { + if (!(x > 90)) { + return x; + } else { + return 90; + } + } else { + return 0; + } +} + +@method_id(102) +fun test2(x: int) { + if (x == 20) { return 20; } + if (x != 50) { return 50; } + if (x == 0) { return 0; } + return -1; +} + +@method_id(103) +fun test3(x: int) { + if (!(x != 20)) { return 20; } + if (!(x == 50)) { return 50; } + if (!x) { return 0; } + return -1; +} + +fun elseif(cond: int) { + if (cond > 0) { + throw(cond); + } +} + +@inline +@method_id(104) +fun test4(x: int): int { + if (x==1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} + +@method_id(105) +fun test5(x: int): (int, int) { + return (test4(x), 222); +} + +@method_id(106) +fun test6(x: int) { + try { + if (x < 0) { return -1; } + elseif (x); + } catch(excNo) { + return excNo * 1000; + } + return 0; +} + +fun doNothing(): void {} + +@method_id(107) +fun test7() { + if (random.uint256()) { return doNothing() } + // here we test that no "missing return" error +} + +fun withNoElse(op: int): int { + if (op == 123) { return 100 } + if (op == 234) { return 200 } + if (op == 345) { return 300 } + throw 0xFF +} + +fun withElse(op: int): int { + if (op == 123) { return 100 } + else if (op == 234) { return 200 } + else if (op == 345) { return 300 } + return 0xFF +} + +fun withMatch(op: int): int { + match (op) { + 123 => return 100, + 234 => return 200, + 345 => return 300, + } + throw 0xFF +} + +@method_id(108) +fun test8(op: int) { + return (withNoElse(op), withElse(op), withMatch(op)); +} + +@noinline +fun demo9_1(x: int) { + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +@noinline +fun demo9_2(x: int) { + if (x > 0) { t.push(123); } +} + +@noinline +fun demo9_3(x: int) { + if (x > 0) { t.push(123); } + return; +} + +@noinline +fun demo9_4(x: int) { + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(109) +fun test9(x: int) { + t = createEmptyTuple(); + demo9_1(x); demo9_2(x); demo9_3(x); demo9_4(x); + return t; +} + +@noinline +fun demo10_1(x: int) { + match(x) { + 0 => t.push(123), + 1 => t.push(456), + } +} + +@noinline +fun demo10_2(x: int) { + if (x != -100) { + match (x) { + 0 => t.push(123), + else => t.push(456) + } + return; + } + t.push(789) +} + +@method_id(110) +fun test10(x: int) { + t = createEmptyTuple(); + demo10_1(x); demo10_2(x); + return t; +} + +@noinline +fun demo_neg_11_1(mutate x: int) { + match (x) { + -1 => { x = 0; } + else => { x = 1; } + } +} + +fun justVoidPush() { + t.push(100); +} + +fun demo_neg_11_2(x: int) { + match (x) { + -1 => t.push(123), + else => t.push(456) + } + return justVoidPush(); +} + +@method_id(111) +fun test11(x: int) { + t = createEmptyTuple(); + demo_neg_11_1(mutate x); + demo_neg_11_2(x); + return (x, t); +} + +fun demo12_1(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +fun demo12_2(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } +} + +fun demo12_3(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + return; +} + +fun demo12_4(x: int) { + __expect_inline(false); + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(112) +fun test12(x: int) { + t = createEmptyTuple(); + demo12_1(x); demo12_2(x); demo12_3(x); demo12_4(x); + return t; +} + +fun main() { + +} + +/** +@testcase | 101 | 0 | 0 +@testcase | 101 | 1000 | 200 +@testcase | 101 | 150 | 100 +@testcase | 101 | -1 | 0 +@testcase | 101 | 87 | 87 +@testcase | 101 | 94 | 90 +@testcase | 102 | 20 | 20 +@testcase | 102 | 40 | 50 +@testcase | 102 | 50 | -1 +@testcase | 103 | 20 | 20 +@testcase | 103 | 40 | 50 +@testcase | 103 | 50 | -1 +@testcase | 104 | 1 | 111 +@testcase | 104 | 3 | 7 +@testcase | 105 | 1 | 111 222 +@testcase | 105 | 3 | 7 222 +@testcase | 106 | -5 | -1 +@testcase | 106 | 5 | 5000 +@testcase | 108 | 123 | 100 100 100 +@testcase | 108 | 345 | 300 300 300 +@testcase | 109 | 10 | [ 123 123 123 123 ] +@testcase | 109 | -10 | [ 456 ] +@testcase | 110 | 0 | [ 123 123 ] +@testcase | 110 | 5 | [ 456 ] +@testcase | 111 | 5 | 1 [ 456 100 ] +@testcase | 112 | 10 | [ 123 123 123 123 ] +@testcase | 112 | -10 | [ 456 ] + +@fif_codegen +""" + test3() PROC:<{ // x + DUP // x x + 20 NEQINT // x '2 + IFNOTJMP:<{ // x + DROP // + 20 PUSHINT // '3=20 + }> // x + DUP // x x + 50 EQINT // x '5 + IFNOTJMP:<{ // x +""" + +@fif_codegen +""" + withNoElse() PROC:<{ + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" + +@fif_codegen +""" + withElse() PROC:<{ + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 8 PUSHPOW2DEC + }> +""" + +@fif_codegen +""" + withMatch() PROC:<{ + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" + +@fif_codegen +""" + demo9_1() PROC:<{ + 0 GTINT + IFJMP:<{ + $t GETGLOB + 123 PUSHINT + TPUSH + $t SETGLOB + }> + $t GETGLOB + 456 PUSHINT + TPUSH + $t SETGLOB + }> +""" + +@fif_codegen +""" + demo9_2() PROC:<{ + 0 GTINT + IFJMP:<{ + $t GETGLOB + 123 PUSHINT + TPUSH + $t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_3() PROC:<{ + 0 GTINT + IFJMP:<{ + $t GETGLOB + 123 PUSHINT + TPUSH + $t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_4() PROC:<{ + DUP + -100 NEQINT + IFJMP:<{ + 0 GTINT + IFJMP:<{ + $t GETGLOB + 123 PUSHINT + TPUSH + $t SETGLOB + }> + }> + DROP + }> +""" + +@fif_codegen +""" + demo10_1() PROC:<{ + DUP + 0 EQINT + IFJMP:<{ + DROP + $t GETGLOB + 123 PUSHINT + TPUSH + $t SETGLOB + }> + 1 EQINT + IFJMP:<{ + $t GETGLOB + 456 PUSHINT + TPUSH + $t SETGLOB + }> + }> +""" + +*/ diff --git a/tolk-tester/tests/if_stmt.tolk b/tolk-tester/tests/if_stmt.tolk deleted file mode 100644 index 0f78a5162..000000000 --- a/tolk-tester/tests/if_stmt.tolk +++ /dev/null @@ -1,66 +0,0 @@ -@method_id(101) -fun test1(x: int): int { - if (x > 200) { - return 200; - } else if (x > 100) { - return 100; - } else if (!(x <= 50)) { - if (!(x > 90)) { - return x; - } else { - return 90; - } - } else { - return 0; - } -} - -@method_id(102) -fun test2(x: int) { - if (x == 20) { return 20; } - if (x != 50) { return 50; } - if (x == 0) { return 0; } - return -1; -} - -@method_id(103) -fun test3(x: int) { - if (!(x != 20)) { return 20; } - if (!(x == 50)) { return 50; } - if (!x) { return 0; } - return -1; -} - -fun main() { - -} - -/** -@testcase | 101 | 0 | 0 -@testcase | 101 | 1000 | 200 -@testcase | 101 | 150 | 100 -@testcase | 101 | -1 | 0 -@testcase | 101 | 87 | 87 -@testcase | 101 | 94 | 90 -@testcase | 102 | 20 | 20 -@testcase | 102 | 40 | 50 -@testcase | 102 | 50 | -1 -@testcase | 103 | 20 | 20 -@testcase | 103 | 40 | 50 -@testcase | 103 | 50 | -1 - -@fif_codegen -""" - test3 PROC:<{ - // x - DUP // x x - 20 NEQINT // x '2 - IFNOTJMP:<{ // x - DROP // - 20 PUSHINT // '3=20 - }> // x - DUP // x x - 50 EQINT // x '5 - IFNOTJMP:<{ // x -""" -*/ diff --git a/tolk-tester/tests/imports-tests.tolk b/tolk-tester/tests/imports-tests.tolk new file mode 100644 index 000000000..c4dc93478 --- /dev/null +++ b/tolk-tester/tests/imports-tests.tolk @@ -0,0 +1,18 @@ +import "imports/some-math" +import "./imports/use-dicts.ext" + +fun main() { + prepareDict_3_30_4_40_5_x(4); + return someAdd(1, 2); +} + +/** +@testcase | 0 | | 3 + +@fif_codegen +""" + main() PROC:<{ + 3 PUSHINT + }> +""" + */ diff --git a/tolk-tester/tests/imports/has-tinf-err.tolk b/tolk-tester/tests/imports/has-tinf-err.tolk new file mode 100644 index 000000000..bc9e5c8b1 --- /dev/null +++ b/tolk-tester/tests/imports/has-tinf-err.tolk @@ -0,0 +1,4 @@ + +fun asdf() { + var i: int = ""; +} diff --git a/tolk-tester/tests/imports/some-helpers.tolk b/tolk-tester/tests/imports/some-helpers.tolk new file mode 100644 index 000000000..a5caa2515 --- /dev/null +++ b/tolk-tester/tests/imports/some-helpers.tolk @@ -0,0 +1 @@ +import "./some-storage" diff --git a/tolk-tester/tests/imports/some-storage.tolk b/tolk-tester/tests/imports/some-storage.tolk new file mode 100644 index 000000000..93a692987 --- /dev/null +++ b/tolk-tester/tests/imports/some-storage.tolk @@ -0,0 +1 @@ +struct SomeStorage {} diff --git a/tolk-tester/tests/imports/use-dicts.tolk b/tolk-tester/tests/imports/use-dicts.ext.tolk similarity index 100% rename from tolk-tester/tests/imports/use-dicts.tolk rename to tolk-tester/tests/imports/use-dicts.ext.tolk diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index e2bd3dd93..af6888df7 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -1,43 +1,63 @@ +type Tup2Int = [int, MInt]; +type Tensor2Int = (int, MInt); +type MInt = int; -fun increment(mutate self: int) { +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; +@pure +fun getBuilderBitsCount(b: builder): int + asm "BBITS"; + +fun MInt.increment(mutate self) { self += 1; } +fun increment(mutate v: MInt) { + v += 1; +} + fun increment2(mutate a: int, mutate b: int) { a += 1; b += 1; } -fun assign1020(mutate a: int, mutate b: int) { +fun assign1020(mutate a: int, mutate b: MInt) { a = 10; b = 20; } -fun plus(mutate self: int, y: int): int { +fun int.plus(mutate self, y: int): int { val newVals = (self + y, y * 10); self = newVals.0; return newVals.1; } +fun plus(mutate x: int, y: int): int { + val newVals = (x + y, y * 10); + x = newVals.0; + return newVals.1; +} + fun eq(v: X): X { return v; } -global gTup: [int]; -global gTens: (int, int); +global gTup: [int] +global gTens: (int, int) @method_id(100) fun testCodegenSimple() { var t1 = [1]; t1.0 = 2; - debugPrintString(""); + debug.printString(""); var t2 = [[1]]; t2.0.0 = 2; - debugPrintString(""); + debug.printString(""); gTup = [1]; gTup.0 = 2; - debugPrintString(""); + debug.printString(""); gTens = (1,2); gTens.1 = 4; - debugPrintString(""); + debug.printString(""); return (t1, t2, gTup, gTens); } @@ -51,7 +71,7 @@ fun test101() { return t; } -global t102: (int, (int, int), [int, int, [int, int]], int); +global t102: (int, (int, MInt), [int, int, Tup2Int], MInt); @method_id(102) fun test102() { @@ -87,8 +107,8 @@ fun test104() { @method_id(105) fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { - var ab = (createEmptyTuple(), (x as int?, y), tupleSize); - ab.0.tuplePush(1); + var ab = (createEmptyTuple(), (x as int?, y), tuple.size); + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -98,8 +118,8 @@ fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { @method_id(106) fun test106(x: int, y: int) { - var ab = [createEmptyTuple(), [x as int?, y], tupleSize]; - ab.0.tuplePush(1); + var ab = [createEmptyTuple(), [x as int?, y], tuple.size]; + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -110,8 +130,8 @@ fun test106(x: int, y: int) { @method_id(107) fun test107() { var ab = createEmptyTuple(); - ab.tuplePush(1); - ab.tuplePush(beginCell().storeInt(1, 32)); + ab.push(1); + ab.push(beginCell().storeInt(1, 32)); return (ab.0 as int, getBuilderBitsCount(ab.1)); } @@ -137,7 +157,7 @@ fun test110(f: int, s: int) { return (x, x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1, x); } -global xx: (int, int); +global xx: Tensor2Int; @method_id(111) fun test111(x: (int, int)) { @@ -154,7 +174,8 @@ fun test112(f: int, s: int) { } @pure -fun getConstTuple() { +@noinline +fun getConstTuple(): Tup2Int { return [1,2]; } @@ -184,7 +205,7 @@ fun test115() { @method_id(116) fun test116() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { return t.100500 as int; } catch(excNo) { @@ -195,9 +216,9 @@ fun test116() { @method_id(117) fun test117() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { - return (t.0 as tuple).0 as int; + return (t.0 as tuple).0 as MInt; } catch(excNo) { return excNo; } @@ -219,20 +240,24 @@ fun getT() { return (1, 2); } @method_id(120) fun test120() { - return (getT().0 = 3, getT().0 = 4, [getT().0 = 5, getT().0 = 6]); + return (getT().0 = 3, getT().0 = sizeof(getT()) * 2, [getT().0 = 5, getT().0 = 6]); } @method_id(121) fun test121(zero: int) { var t = createEmptyTuple(); - t.tuplePush(-100); - t.tupleSetAt(0, zero); + t.push(-100); + t.set(0, zero); (t.0 as int).increment(); (((t.0) as int) as int).increment(); increment(mutate t.0 as int); return t; } +fun (T1, T2).isFirstComponentGt0(self): bool { + return self.0 > 0; +} + fun isFirstComponentGt0(t: (T1, T2)): bool { return t.0 > 0; } @@ -241,7 +266,7 @@ fun isFirstComponentGt0(t: (T1, T2)): bool { fun test122(x: (int, int)) { return ( isFirstComponentGt0(x), isFirstComponentGt0((2, beginCell())), isFirstComponentGt0((0, null)), - x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null).isFirstComponentGt0() + x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null as slice?).isFirstComponentGt0() ); } @@ -282,8 +307,7 @@ fun main(){} @fif_codegen """ - testCodegenSimple PROC:<{ - // + testCodegenSimple() PROC:<{ 1 PUSHINT // '2=1 SINGLE // t1 2 PUSHINT // t1 '3=2 @@ -303,45 +327,43 @@ fun main(){} STRDUMP DROP 1 PUSHINT // t1 t2 '20=1 SINGLE // t1 t2 '18 - gTup SETGLOB + $gTup SETGLOB 2 PUSHINT // t1 t2 '21=2 - gTup GETGLOB // t1 t2 '21=2 g_gTup + $gTup GETGLOB // t1 t2 '21=2 g_gTup SWAP // t1 t2 g_gTup '21=2 0 SETINDEX // t1 t2 g_gTup - gTup SETGLOB + $gTup SETGLOB x{} PUSHSLICE // t1 t2 '25 STRDUMP DROP 1 PUSHINT // t1 t2 '28=1 2 PUSHINT // t1 t2 '26=1 '27=2 PAIR - gTens SETGLOB + $gTens SETGLOB 4 PUSHINT // t1 t2 g_gTens.1=4 - gTens GETGLOB + $gTens GETGLOB UNPAIR // t1 t2 g_gTens.1=4 g_gTens.0 g_gTens.1 DROP // t1 t2 g_gTens.1=4 g_gTens.0 SWAP // t1 t2 g_gTens.0 g_gTens.1=4 PAIR - gTens SETGLOB + $gTens SETGLOB x{} PUSHSLICE // t1 t2 '36 STRDUMP DROP - gTup GETGLOB // t1 t2 g_gTup - gTens GETGLOB + $gTup GETGLOB // t1 t2 g_gTup + $gTens GETGLOB UNPAIR // t1 t2 g_gTup g_gTens.0 g_gTens.1 }> """ @fif_codegen """ - testCodegenNoPureIndexedAccess PROC:<{ - // + testCodegenNoPureIndexedAccess() PROC:<{ 0 PUSHINT // '8=0 }> """ @fif_codegen """ - testCodegenIndexPostfix1 PROC:<{ - // x.0 x.1 + testCodegenIndexPostfix1() PROC:<{ // x.0 x.1 // ab.1 ab.0 SWAP // ab.0 ab.1 }> @@ -349,8 +371,7 @@ fun main(){} @fif_codegen """ - testCodegenIndexPostfix2 PROC:<{ - // x.0 x.1.0 x.1.1 x.2 + testCodegenIndexPostfix2() PROC:<{ // x.0 x.1.0 x.1.1 x.2 s2 POP // y.0 y.2 y.1.1 s1 s2 XCHG // y.2 y.0 y.1.1 }> diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 5020d0ddc..9c836baf4 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -5,7 +5,7 @@ fun eq(value: X): X { return value; } fun test1(x: int, y: int) { __expect_type(0, "int"); - __expect_type("0"c, "int"); + __expect_type(stringCrc32("0"), "int"); __expect_type(x, "int"); __expect_type(x + y, "int"); __expect_type(x * y, "int"); @@ -15,7 +15,7 @@ fun test1(x: int, y: int) { __expect_type(x = x, "int"); __expect_type(x += x, "int"); __expect_type(x &= x, "int"); - __expect_type(random() ? x : y, "int"); + __expect_type(random.uint256() ? x : y, "int"); __expect_type(eq(x), "int"); __expect_type(eq(x), "int"); __expect_type(eq(null), "int?"); @@ -30,6 +30,15 @@ fun test1(x: int, y: int) { __expect_type(beginCell(), "builder"); __expect_type(beginCell().endCell(), "cell"); } + __expect_type((1, 2), "(int, int)"); + __expect_type((1, 2, ), "(int, int)"); + __expect_type((1, ), "int"); + __expect_type((((1,)),), "int"); + __expect_type([1, 2], "[int, int]"); + __expect_type([1, 2, ], "[int, int]"); + __expect_type([1, ], "[int]"); + __expect_type([((1, ), ), ], "[int]"); + __expect_type([[[1, ], ], ], "[[[int]]]"); } fun test2(x: int, y: bool) { @@ -38,7 +47,7 @@ fun test2(x: int, y: bool) { __expect_type(x <= x, "bool"); __expect_type(x <=> x, "bool"); __expect_type(x <=> x, "bool"); - __expect_type(!random(), "bool"); + __expect_type(!random.uint256(), "bool"); __expect_type(!!(x != null), "bool"); __expect_type(x ? x != null : null == x, "bool"); __expect_type(y & true, "bool"); @@ -51,7 +60,7 @@ fun test2(x: int, y: bool) { fun test3() { __expect_type(true as int, "int"); - __expect_type(!random() as int, "int"); + __expect_type(!random.uint256() as int, "int"); } fun test4(x: int) { @@ -66,13 +75,13 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell?]"); + __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, dict]"); } fun test6() { var t = createEmptyTuple(); __expect_type(t, "tuple"); - t.tuplePush(1); + t.push(1); __expect_type(t, "tuple"); } @@ -86,6 +95,15 @@ fun test7() { // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)"); } +fun unnamed_args(_: int, _: slice, _: int) { + return true; +} + +@method_id(108) +fun test8(x: int, z: int) { + return unnamed_args(x, "asdf", z); +} + fun alwaysThrows(): never { throw 123; } fun alwaysThrowsNotAnnotated() { throw 123; } fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } @@ -103,5 +121,6 @@ fun main() { } /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 108 | 1 10 | -1 */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk new file mode 100644 index 000000000..d85c5543b --- /dev/null +++ b/tolk-tester/tests/inline-tests.tolk @@ -0,0 +1,557 @@ +fun foo1(x: int): int { + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 1; + } + return 2; +} + +@inline +fun foo2(x: int): int { + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 11; + } + return 22; +} + +@inline_ref +fun foo3(x: int): int { + if (x == 1) { + return 111; + } + return 222; +} + +@inline_ref +fun foo_light_but_ref() { + __expect_inline(false); + return 0; +} + +@noinline +fun foo_light_but_no() { + __expect_inline(false); + return 0; +} + +@method_id(101) +fun test1(x: int): (int, int, int) { + __expect_inline(false); // @method_id + return (foo1(x) + 1, foo2(x) + 1, foo3(x) + 1 + foo_light_but_ref() + foo_light_but_no()); +} + +global g: int; + +@inline +fun foo_repeat() { + __expect_inline(true); + g = 1; + repeat (5) { + g *= 2; + } +} + +@inline +fun foo_until(): int { + __expect_inline(true); + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; +} + +@inline +fun foo_while(): int { + __expect_inline(true); + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +@method_id(102) +fun test2() { + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); +} + +@inline +fun foo_big(x: int): int { + __expect_inline(true); + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +@noinline +fun foo_big_not_annotated_called_once(x: int): int { + __expect_inline(true); // called only once, inline + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +fun foo_big_not_annotated_called_twice(x: int): int { + __expect_inline(false); // too big for in-place auto-inlining when called multiple times + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +@method_id(103) +fun test3(x: int): int { + return foo_big(x) * 10 + 5; +} + +@method_id(104) +fun test4(x: int) { + return (foo_big_not_annotated_called_once(x) * 10, foo_big_not_annotated_called_twice(x) * 10, foo_big_not_annotated_called_twice(x)); +} + +struct Storage { + owner: address; + lastCall: int64; + extra: Cell<(int8, uint256)>; +} + +fun Storage.load() { + __expect_inline(true); + return Storage.fromCell(contract.getData()); +} + +fun Storage.save(self) { + __expect_inline(true); + contract.setData(self.toCell()); +} + +@method_id(105) +fun test5() { + var st = Storage.load(); + st.lastCall = 0; + st.save(); +} + +@inline +fun factorial(x: int): int { + __expect_inline(false); // due to recursion + if (x <= 1) { + return 1; + } + return x * factorial(x - 1); +} + +fun recurse1(x: int): int { __expect_inline(false); return recurse2(x + 1); } +fun recurse2(x: int): int { __expect_inline(false); return recurse3(x + 1); } +fun recurse3(x: int): int { __expect_inline(false); return recurse1(x + 1); } + +global g10: int; + +fun startCellWith(x: int) { + __expect_inline(true); + return beginCell().storeUint(x, 32); +} + +fun startCellWithG10() { + __expect_inline(true); + return beginCell().storeUint(g10, 32); +} + +fun builder.myBitsCount(self): int { + __expect_inline(true); + return self.bitsCount(); +} + +fun builder.store32(mutate self, v: int32): self { + __expect_inline(true); + return self.storeUint(v, 32); +} + +fun sum(a: int, b: int) { + __expect_inline(true); + return a + b; +} + +fun eq(v: T): T { + __expect_inline(true); + return v; +} + +fun get20() { return 20; } + +struct Point { + x: int; + y: int; +} + +fun Point.getX(self) { + __expect_inline(true); + return self.x; +} + +fun Point.create(x: int, y: int): Point { + __expect_inline(true); + return {x, y} +} + +fun increment(mutate x: int) { + __expect_inline(true); + x += 1; +} + +@method_id(106) +fun test6() { + var p = Point.create(10, 20); + increment(mutate p.x); + p.y = p.getX(); + return p; +} + +@method_id(107) +fun test7() { + g10 = 10; + return startCellWithG10().store32(20).myBitsCount(); +} + +@method_id(108) +fun test8() { + var x = 0; + return (startCellWith(x = 5).store32(x += 20).store32(x), x); +} + +@method_id(109) +fun test9(x: int) { + var p1: Point = {x, y: 0}; + var p2: Point = {x: 0, y: p1.getX()}; + var t = createEmptyTuple(); + t.pushPoint(p1); + t.pushPoint(p2); + return (t.popPoint(), t.popPoint(), t); +} + +fun tuple.pushPoint(mutate self, p: Point) { + __expect_inline(true); + var x = p.x; + self.push(x); + self.push(p.y); +} + +fun tuple.popPoint(mutate self): Point { + __expect_inline(true); + var y = self.pop(); + return {x: self.pop(), y}; +} + +@method_id(110) +fun test10() { + var (a, b, c) = usedIn10ButDeclaredBelow(5); + return (a, b, c.bitsCount()); +} + +global g11_1: int; +global g11_2: int; + +fun mutateGlobals(mutate v: int, inc11_2: int) { + __expect_inline(true); + v += 1; + g11_2 += inc11_2; +} + +@method_id(111) +fun test11() { + g11_1 = 0; + g11_2 = 0; + mutateGlobals(mutate g11_1, 5); // 1 5 + mutateGlobals(mutate g11_1, g11_2); // 2 10 + mutateGlobals(mutate g11_2, g11_1); // 2 11 + return (g11_1, g11_2); +} + +fun nested1(x: int) { + var y = x + 2; + x += 7; + return y; +} + +fun nested2(x: int) { + var y = x + 2; + var z = nested1(y); + y += z; + return y + z; +} + +@method_id(112) +fun test12(x: int) { + var r1 = nested2(10); + var r2 = nested2(x); + return (r1, r2); +} + +fun anotherMath(arg: int) { + __expect_inline(true); + var cp_x = arg; + increment(mutate arg); + return cp_x + arg; +} + +@method_id(113) +fun test13(p: Point) { + anotherMath(p.x); + var r = anotherMath(p.y); + return (p, r); +} + +fun evalPoint(p: Point) { + __expect_inline(true); + increment(mutate p.x); + p.y *= 2; + return max(p.x, p.y); +} + +@method_id(114) +fun test14(p2: Point) { + var p1: Point = { x: 0, y: 0 }; + var r1 = evalPoint(p1); + return (r1, p1, evalPoint(p2), p2); +} + +global t: tuple; + +fun logT(a: int) { + __expect_inline(true); + t.push(a); +} + +fun demoInlined1(x: int) { + if (x == -1) { + logT(10); + } else if (x == 0) { + logT(20); + } +} + +fun demoInlined2(x: int) { + __expect_inline(true); + match (x) { + -1 => logT(10), + else => logT(20), + } +} + +fun wrapperNotInlined(x: int?) { + if (x != null) { + demoInlined1(x); + return; + } + logT(100); +} + +fun someUsages(x: int?) { + __expect_inline(true); + wrapperNotInlined(x); + demoInlined2(x!); + return; +} + +@method_id(115) +fun test15(x: int?) { + t = createEmptyTuple(); + someUsages(x); + return t; +} + +struct (0x01) A16 { a: int8; } +struct (0x02) B16 {} +struct (0x03) C16 {} +type U16 = A16 | B16 | C16; + +fun check16(msg: U16) { + match (msg) { + A16 => { assert(msg.a > 0, 100); } + B16 => {} + C16 => {} + } +} + +fun wrap16(msg: U16, check: bool) { + if (check) { + check16(msg); + return; + } + throw 123; +} + +@method_id(116) +fun test16() { + wrap16(U16.fromSlice(stringHexToSlice("0170")), true); + return -1; +} + + +fun main() { + // regardless of number of calls, it will be inlined, since it's lightweight in AST terms + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + usedIn10ButDeclaredBelow(10); +} + +fun usedIn10ButDeclaredBelow(x: int) { + __expect_inline(true); + var y = eq(x); + return (x, y, eq(startCellWith(x))); +} + + + +/** + method_id | in | out +@testcase | 101 | 1 | 2 12 112 +@testcase | 101 | 2 | 3 23 223 +@testcase | 102 | | 32 256 1024 +@testcase | 103 | 9 | 911111111111111111111111111111111111111111111111115 +@testcase | 104 | 9 | 911111111111111111111111111110 911111111111111111111111111110 91111111111111111111111111111 +@testcase | 106 | | 11 11 +@testcase | 107 | | 64 +@testcase | 108 | | BC{0018000000050000001900000019} 25 +@testcase | 109 | 10 | 0 10 10 0 [] +@testcase | 110 | | 5 5 32 +@testcase | 111 | | 2 11 +@testcase | 112 | 10 | 40 40 +@testcase | 113 | 10 20 | 10 20 41 +@testcase | 114 | 10 20 | 1 0 0 40 10 20 +@testcase | 115 | -1 | [ 10 10 ] +@testcase | 115 | 1 | [ 20 ] +@testcase | 116 | | -1 + +@fif_codegen_avoid Storage.load +@fif_codegen_avoid Storage.save +@fif_codegen +""" + test5() PROC:<{ // + c4 PUSH // '7 + CTOS // s + LDMSGADDR // '11 s +""" + +@fif_codegen +""" + test6() PROC:<{ + 11 PUSHINT + DUP + }> +""" + +@fif_codegen +""" + test8() PROC:<{ // + NEWC + x{00000005} STSLICECONST // '3 + 25 PUSHINT // '3 '9 + SWAP // x self + x{0000001900000019} STSLICECONST // x '3 + SWAP // '3 x + }> +""" + +@fif_codegen +""" + test9() PROC:<{ // x + 0 PUSHINT // p1.x p1.y=0 + s0 s1 PUSH2 // p1.x p1.y=0 p2.x=0 p2.y + NIL // x p1.y=0 p2.x=0 p2.y self + s0 s4 XCHG2 // p2.y p1.y=0 p2.x=0 self x + TPUSH // p2.y p1.y=0 p2.x=0 self + ROT // p2.y p2.x=0 self p1.y=0 + TPUSH // p2.y x=0 self + SWAP // p2.y self x=0 + TPUSH // p2.y self + SWAP // self p2.y + TPUSH // self + TPOP // self y + SWAP // y self + TPOP // '36 self '35 + SWAP // '36 '35 self + TPOP // '36 '35 self y + SWAP // '36 '35 y self + TPOP // '36 '35 '50 t '49 + s3 s4 XCHG // '35 '36 '50 t '49 + -ROT // '35 '36 '49 '50 t + }> +""" + +@fif_codegen +""" + test10() PROC:<{ + 5 PUSHINT // '3=5 + DUP // '3=5 y=5 + NEWC + x{00000005} STSLICECONST // a=5 b=5 c + BBITS // a=5 b=5 '18 + }> +""" + +@fif_codegen +""" + test12() PROC:<{ // x + 40 PUSHINT // x r1 + SWAP // r1 x + 2 ADDCONST // r1 y + ... +""" + +@fif_codegen +""" + wrapperNotInlined() PROC:<{ // x + DUP // x x + ISNULL // x '1 + IFNOTJMP:<{ // x + DUP // x x + -1 EQINT // x '3 + IFJMP:<{ // x +""" + +@fif_codegen +""" + wrap16() PROC:<{ + IFJMP:<{ + 139 PUSHINT + OVER + EQUAL + IFJMP:<{ + DROP + 0 GTINT + 100 THROWIFNOT + }> + 140 PUSHINT + s2 POP + EQUAL + IFJMP:<{ + }> + }> + 123 THROW + }> +""" +*/ diff --git a/tolk-tester/tests/inline_big.tolk b/tolk-tester/tests/inline_big.tolk deleted file mode 100644 index be014eb5d..000000000 --- a/tolk-tester/tests/inline_big.tolk +++ /dev/null @@ -1,62 +0,0 @@ -@inline -fun foo(x: int): int { - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - return x; -} - -fun main(x: int): int { - return foo(x) * 10 + 5; -} -/** - method_id | in | out -@testcase | 0 | 9 | 9111111111111111111111111111111111111111111111111115 -*/ diff --git a/tolk-tester/tests/inline_if.tolk b/tolk-tester/tests/inline_if.tolk deleted file mode 100644 index 9f1fa8c12..000000000 --- a/tolk-tester/tests/inline_if.tolk +++ /dev/null @@ -1,28 +0,0 @@ -fun foo1(x: int): int { - if (x == 1) { - return 1; - } - return 2; -} -@inline -fun foo2(x: int): int { - if (x == 1) { - return 11; - } - return 22; -} -@inline_ref -fun foo3(x: int): int { - if (x == 1) { - return 111; - } - return 222; -} -fun main(x: int): (int, int, int) { - return (foo1(x)+1, foo2(x)+1, foo3(x)+1); -} -/** - method_id | in | out -@testcase | 0 | 1 | 2 12 112 -@testcase | 0 | 2 | 3 23 223 -*/ diff --git a/tolk-tester/tests/inline_loops.tolk b/tolk-tester/tests/inline_loops.tolk deleted file mode 100644 index eba595a5e..000000000 --- a/tolk-tester/tests/inline_loops.tolk +++ /dev/null @@ -1,48 +0,0 @@ -global g: int; - -@inline -fun foo_repeat() { - g = 1; - repeat(5) { - g *= 2; - } -} - -@inline -fun foo_until(): int { - g = 1; - var i: int = 0; - do { - g *= 2; - i += 1; - } while (i < 8); - return i; -} - -@inline -fun foo_while(): int { - g = 1; - var i: int = 0; - while (i < 10) { - g *= 2; - i += 1; - } - return i; -} - -fun main() { - foo_repeat(); - var x: int = g; - foo_until(); - var y: int = g; - foo_while(); - var z: int = g; - return (x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | | 32 256 1024 - -@code_hash 102749806552989901976653997041637095139193406161777448419603700344770997608788 -*/ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk new file mode 100644 index 000000000..cd557092e --- /dev/null +++ b/tolk-tester/tests/intN-tests.tolk @@ -0,0 +1,262 @@ +fun takeAnyInt(a: int) { } +fun getAnyInt(): int { return 0; } +fun getNullableInt32(): int32? { return 32; } +fun getNullableVarInt32(): varint32? { return 32; } + +const someN_u128: uint128 = 128; +const someN_i32: int32 = 20 + (12 as int32) * (1 as uint8); +const someN_coins = 10 as coins; + +const cost1 = ton("1.234"); +const cost2 = ton("0.05") + ton("0.001"); + +fun rand(): uint256 { return random.uint256(); } + +fun autoInferInt8(x: int8) { + if (rand()) { return x; } + else if (rand()) { return 0 as int8; } + else if (rand()) { return x!; } + else { return x += x; } +} + +fun autoInferUInt16Opt(x: uint16) { + if (rand()) { return autoInferInt8(x as int8) as uint16; } + return null; +} + +fun inferInt_v1(): int { + if (rand()) { return 0; } + else if (rand()) { return 0 as uint16; } + else { return 0 as int8; } +} + +fun inferInt_v2(): int { + if (rand()) { return 0 as uint1; } + else if (rand()) { return 0 as int123; } + else { return 0 as int8; } +} + +@method_id(101) +fun test1(x: int8) { + var y: int8 = x; + y += x; + x *= y; + x = 1000000; + y = x; + __expect_type(x / y, "int"); + __expect_type(x & y, "int"); + __expect_type(x != y, "bool"); + if (x != y) { return -1; } + if (!(y == x)) { return -1; } + + var a1: uint8 = 0; + var a2: uint8? = 0; + var a3: uint8? = 0 as int?; + __expect_type(a1, "uint8"); + __expect_type(a2, "uint8"); + __expect_type(a3, "uint8?"); + + var z = x + y; + __expect_type(z, "int"); + __expect_type(someN_u128, "uint128"); + __expect_type(someN_coins, "coins"); + return x / y + someN_u128; +} + +fun test2(): (uint8, uint8, uint8) { + var x: uint1 = 1; + return (1, x as uint8, x as int); +} + +fun test3(op: int32, qid: uint64) { + op = qid as int32; + op = someN_i32; + + op + qid; + op = op + qid; + op = op & qid; + op += qid; + op &= qid; + if (op == qid) {} + if ((op as int32?)! == qid) {} + if (op == (qid as uint64)) {} + if ((op as int257?)! != (qid as uint256?)!) {} + __expect_type(op << qid, "int"); + + takeAnyInt(op); + op = getAnyInt(); + + __expect_type(autoInferInt8, "(int8) -> int8"); + __expect_type(autoInferUInt16Opt(0), "uint16?"); + __expect_type(inferInt_v1(), "int"); + __expect_type(inferInt_v2(), "int"); + + var amount: uint100 = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; +} + +@method_id(104) +fun test4(): (int32, int32?, bool, bool) { + var x = getNullableInt32(); + __expect_type(x, "int32?"); + if (x! != x! || x! != 32) { + return (0, null, false, false); + } + if (x == null) { + return (-1, null, false, false); + } + x += x; + return (x, x, x == x, x == getAnyInt()); +} + +@method_id(105) +fun test5() { + var (x: int8, y: uint16) = (1, 2); + var cell = beginCell().storeInt(x, 32).storeInt(y, 32 as int32).endCell(); + var slice = cell.beginParse(); + x = slice.loadInt(32); + y = slice.loadInt(32); + __expect_type(x & y, "int"); + return (x + y, x && y, x & y); +} + +fun test6() { + var x: int11 = ~(0 as int22); + while (x) {} + return x ? x as int : 0; +} + +fun test7() { + var n = getNullableVarInt32(); + __expect_type(n!, "varint32"); + __expect_type(n, "varint32?"); + if (n != null) { + __expect_type(n, "varint32"); + __expect_type(n += n, "varint32"); + __expect_type(n += n as int8, "varint32"); + } else { + n = 0; + n = 0 as varint32?; + n = 0 as varint32; + } + __expect_type(n, "varint32"); + __expect_type(n as uint32, "uint32"); + __expect_type(n as varint16, "varint16"); + __expect_type((n as varint16) as int, "int"); +} + +fun test8(amount1: coins, amount2: coins) { + __expect_type(amount1 + 0, "coins"); + __expect_type(amount1 + amount2, "coins"); + __expect_type(amount1 += amount2, "coins"); + __expect_type(amount1 - 10, "coins"); + __expect_type(amount1 &= 1, "coins"); + __expect_type(amount1 &= (1 as coins), "coins"); + + __expect_type(amount1 & 1, "int"); + __expect_type(amount1 * 10, "int"); + __expect_type(amount1 << amount2, "int"); +} + +@method_id(109) +fun test9(c: coins, i: int8) { + __expect_type(c as int8, "int8"); + __expect_type(i as coins, "coins"); + __expect_type(c + i, "coins"); + + var amount: coins = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; + __expect_type(amount, "coins"); + if (!amount) { while (amount) { amount -= 1; } } + + takeAnyInt(amount); + takeAnyInt(percent); + takeAnyInt(true as int3); + takeAnyInt(amount as int3); + + return (amount, (true as int8) as coins); +} + +@method_id(110) +fun test10() { + return (ton("0.05"), ton("0.05") + 100, cost1, cost2); +} + +@method_id(111) +fun test11() { + __expect_type(ton("0"), "coins"); + __expect_type(ton("0") + ton("0"), "coins"); + __expect_type(ton("0") * 20, "int"); + return [ + ton("1"), + ton("1.0"), + ton("1.00000"), + ton("-321.123456789"), + ton("+321.123456789876"), + ton("0001.1000") + ]; +} + +fun test12() { + var a = ton("1") + ton("2"); // check absence in fif codegen + return ton("0.1"); +} + +fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, int32?, int32?) { + return (x1, x2, x3, x4, x4 as int32, x4 as int32?); +} + +@method_id(114) +fun test14(firstComponent: int8?): (int, int16)? { + return firstComponent! < 10 ? null : (firstComponent! as int, 2 as int16); +} + +@noinline +fun assign0(mutate v: T) { v = 0; } + +fun main() { + var t = createEmptyTuple(); + t.push(1); + t.push(2); + t.push(3); + assign0(mutate t.0 as int8); + assign0(mutate t.1 as uint16); + assign0(mutate t.2 as int); + __expect_type(t.0 as int1, "int1"); + __expect_type(t.0 as int257, "int257"); + __expect_type(0 as int32, "int32"); + __expect_type((0 as int32) as uint64?, "uint64?"); + __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); + __expect_type(0 as coins, "coins"); + __expect_type(someN_coins as coins?, "coins?"); + __expect_type(someN_coins as int8?, "int8?"); + __expect_type(10>3 ? (0 as uint8) as uint8 | uint16 : 0 as uint16, "uint8 | uint16"); + return t; +} + +/** +@testcase | 0 | | [ 0 0 0 ] +@testcase | 101 | 0 | 129 +@testcase | 104 | | 64 64 -1 0 +@testcase | 105 | | 3 -1 0 +@testcase | 109 | 4 4 | 500 -1 +@testcase | 110 | | 50000000 50000100 1234000000 51000000 +@testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] +@testcase | 114 | 5 | (null) (null) 0 +@testcase | 114 | 15 | 15 2 typeid-2 + +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() + +@fif_codegen +""" + test12() PROC:<{ + 100000000 PUSHINT // '4=100000000 + }> +""" +*/ diff --git a/tolk-tester/tests/invalid-builtin-1.tolk b/tolk-tester/tests/invalid-builtin-1.tolk deleted file mode 100644 index 6a7f1ca7f..000000000 --- a/tolk-tester/tests/invalid-builtin-1.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun moddiv2(x: int, y: int): (int, int) builtin; - -/** -@compilation_should_fail -@stderr -""" -`builtin` used for non-builtin function -fun moddiv2 -""" -*/ diff --git a/tolk-tester/tests/invalid-call-1.tolk b/tolk-tester/tests/invalid-call-1.tolk deleted file mode 100644 index 7435bb3cf..000000000 --- a/tolk-tester/tests/invalid-call-1.tolk +++ /dev/null @@ -1,10 +0,0 @@ -const asdf = 1; - -fun main(x: int) { - return x.asdf(); -} - -/** -@compilation_should_fail -@stderr non-existing method `asdf` of type `int` - */ diff --git a/tolk-tester/tests/invalid-call-4.tolk b/tolk-tester/tests/invalid-call-4.tolk deleted file mode 100644 index c8f7dcebf..000000000 --- a/tolk-tester/tests/invalid-call-4.tolk +++ /dev/null @@ -1,13 +0,0 @@ -fun methodWith1Param(self: int, param: int) { - -} - -fun main() { - val x = 10; - x.methodWith1Param(2, "asdf"); -} - -/** -@compilation_should_fail -@stderr too many arguments in call to `methodWith1Param`, expected 1, have 2 - */ diff --git a/tolk-tester/tests/invalid-call-6.tolk b/tolk-tester/tests/invalid-call-6.tolk deleted file mode 100644 index cbf598066..000000000 --- a/tolk-tester/tests/invalid-call-6.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun nothing() { -} - -fun main() { - val x = 0; - return x.nothing(); -} - -/** -@compilation_should_fail -@stderr `nothing` has no parameters and can not be called as method - */ diff --git a/tolk-tester/tests/invalid-cmt-old.tolk b/tolk-tester/tests/invalid-cmt-old.tolk deleted file mode 100644 index 58927d3a0..000000000 --- a/tolk-tester/tests/invalid-cmt-old.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun main(): int { - ;; here is not a comment -} - -/** -@compilation_should_fail -@stderr error: expected `;`, got `is` - */ diff --git a/tolk-tester/tests/invalid-declaration-6.tolk b/tolk-tester/tests/invalid-declaration-6.tolk deleted file mode 100644 index 42cb7b953..000000000 --- a/tolk-tester/tests/invalid-declaration-6.tolk +++ /dev/null @@ -1,8 +0,0 @@ -get seqno(self: int) { - return 0; -} - -/** -@compilation_should_fail -@stderr get methods can't have `mutate` and `self` params - */ diff --git a/tolk-tester/tests/invalid-declaration/err-1043.tolk b/tolk-tester/tests/invalid-declaration/err-1043.tolk new file mode 100644 index 000000000..4ef7d292f --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1043.tolk @@ -0,0 +1,6 @@ +get id() { return 123; } + +/** +@compilation_should_fail +@stderr expected `fun`, got `id` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1055.tolk b/tolk-tester/tests/invalid-declaration/err-1055.tolk new file mode 100644 index 000000000..3b89f53e0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1055.tolk @@ -0,0 +1,19 @@ + +struct Options { + o1: bool; + o2: bool; +} + +fun getBool() { return true; } + +fun f(x: Options = { + o1: true, + o2: getBool(), +}) { +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr o2: getBool() + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1074.tolk b/tolk-tester/tests/invalid-declaration/err-1074.tolk new file mode 100644 index 000000000..b3298436f --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1074.tolk @@ -0,0 +1,6 @@ +fun Container.create() {} + +/** +@compilation_should_fail +@stderr unknown type name `Container` + */ diff --git a/tolk-tester/tests/invalid-declaration-8.tolk b/tolk-tester/tests/invalid-declaration/err-1084.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-8.tolk rename to tolk-tester/tests/invalid-declaration/err-1084.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1140.tolk b/tolk-tester/tests/invalid-declaration/err-1140.tolk new file mode 100644 index 000000000..ab902fc26 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1140.tolk @@ -0,0 +1,27 @@ +fun okWithJustInt(a: int8, b: int16): int { + if (a > 10) { + return a; + } + return b; +} + +fun okWithManualUnion(a: int8, b: int16): int16 | int8 { + if (a > 10) { + return a; + } + return b; +} + +fun autoInferUnionType(a: int8, b: int16) { + if (a > 10) { + return a; + } + return b; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected +@stderr declare `fun (...): ` manually +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1153.tolk b/tolk-tester/tests/invalid-declaration/err-1153.tolk new file mode 100644 index 000000000..1dae9c9b2 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1153.tolk @@ -0,0 +1,9 @@ +fun foo() { return 1; } + +const asdf = 1 + foo(); + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr const asdf + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1190.tolk b/tolk-tester/tests/invalid-declaration/err-1190.tolk new file mode 100644 index 000000000..510e1d1cc --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1190.tolk @@ -0,0 +1,9 @@ +fun main() { + var i; +} + +/** +@compilation_should_fail +@stderr provide a type for a variable, because its default value is omitted: +@stderr var i: ; + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1193.tolk b/tolk-tester/tests/invalid-declaration/err-1193.tolk new file mode 100644 index 000000000..45124998e --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1193.tolk @@ -0,0 +1,7 @@ +fun int.smth(self: int) { +} + +/** +@compilation_should_fail +@stderr `self` parameter should not have a type + */ diff --git a/tolk-tester/tests/invalid-declaration-1.tolk b/tolk-tester/tests/invalid-declaration/err-1209.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1209.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1226.tolk b/tolk-tester/tests/invalid-declaration/err-1226.tolk new file mode 100644 index 000000000..12620289c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1226.tolk @@ -0,0 +1,14 @@ +type MInt = int; +type MInt2 = MInt; + +fun int.check(self) {} +fun MInt.check(self) {} +fun MInt2.check(self) {} +fun `MInt`.`check`(self) {} + +/** +@compilation_should_fail +@stderr redefinition of symbol +@stderr err-1226.tolk:7 +@stderr err-1226.tolk:5 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1285.tolk b/tolk-tester/tests/invalid-declaration/err-1285.tolk new file mode 100644 index 000000000..c647a1535 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1285.tolk @@ -0,0 +1,14 @@ +struct Pair { + first: T1; + second: T2; +} + +struct DoesntWork { + p: Pair; +} + +/** +@compilation_should_fail +@stderr struct `Pair` expects 2 type arguments, but 1 provided +@stderr p: Pair + */ diff --git a/tolk-tester/tests/invalid-declaration-10.tolk b/tolk-tester/tests/invalid-declaration/err-1361.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-10.tolk rename to tolk-tester/tests/invalid-declaration/err-1361.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1378.tolk b/tolk-tester/tests/invalid-declaration/err-1378.tolk new file mode 100644 index 000000000..bd2750fcb --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + value: T; +} + +type ErrWrapper = int | Wrapper; + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr int | Wrapper; + */ diff --git a/tolk-tester/tests/invalid-get-method-2.tolk b/tolk-tester/tests/invalid-declaration/err-1449.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1449.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1458.tolk b/tolk-tester/tests/invalid-declaration/err-1458.tolk new file mode 100644 index 000000000..c1d24dfa2 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1458.tolk @@ -0,0 +1,13 @@ +struct Pair { + first: A; + second: B; +} + +type PairAlias = Pair; + +fun PairAlias.create() {} + +/** +@compilation_should_fail +@stderr type `PairAlias` expects 2 type arguments, but 1 provided + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1473.tolk b/tolk-tester/tests/invalid-declaration/err-1473.tolk new file mode 100644 index 000000000..ef79b5033 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1473.tolk @@ -0,0 +1,16 @@ +fun getIntOrSlice() { return 5 as int | slice; } + +fun autoInferUnionType(a: int | slice | null) { + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return beginCell(), + null => return -20 + }; + return cc; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `builder | int | slice`; probably, it's not what you expected +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1483.tolk b/tolk-tester/tests/invalid-declaration/err-1483.tolk new file mode 100644 index 000000000..01812fedc --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1483.tolk @@ -0,0 +1,8 @@ +type A = B; +type B = (int, A); + +/** +@compilation_should_fail +@stderr type `A` circularly references itself +@stderr type A = B + */ diff --git a/tolk-tester/tests/invalid-mutate-13.tolk b/tolk-tester/tests/invalid-declaration/err-1505.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-13.tolk rename to tolk-tester/tests/invalid-declaration/err-1505.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1556.tolk b/tolk-tester/tests/invalid-declaration/err-1556.tolk new file mode 100644 index 000000000..831ab2d49 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1556.tolk @@ -0,0 +1,7 @@ +fun nothing(self) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1585.tolk b/tolk-tester/tests/invalid-declaration/err-1585.tolk new file mode 100644 index 000000000..5abf00017 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1585.tolk @@ -0,0 +1,15 @@ +struct B { a: A } +struct A { b: BAlias } + +type BAlias = B; + +fun test(a: A) { + a.b; + a.b.a; +} + +/** +@compilation_should_fail +(error message not stable, sometimes about A, sometimes B, it's okay, they are in a hashmap) +@stderr size is infinity due to recursive fields + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1617.tolk b/tolk-tester/tests/invalid-declaration/err-1617.tolk new file mode 100644 index 000000000..dbfe89719 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1617.tolk @@ -0,0 +1,8 @@ +fun main() { + val (s, i); +} + +/** +@compilation_should_fail +@stderr variables declaration must be followed by assignment + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1663.tolk b/tolk-tester/tests/invalid-declaration/err-1663.tolk new file mode 100644 index 000000000..0c41e4c2c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1663.tolk @@ -0,0 +1,11 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +/** +@compilation_should_fail +@stderr duplicate generic parameter `T` +@stderr fun Container.wrap + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1665.tolk b/tolk-tester/tests/invalid-declaration/err-1665.tolk new file mode 100644 index 000000000..4ba5297b9 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1665.tolk @@ -0,0 +1,8 @@ +fun f(a: int, b: int = a) { + +} + +/** +@compilation_should_fail +@stderr symbol `a` is not a constant + */ diff --git a/tolk-tester/tests/invalid-redefinition-1.tolk b/tolk-tester/tests/invalid-declaration/err-1683.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1683.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1701.tolk b/tolk-tester/tests/invalid-declaration/err-1701.tolk new file mode 100644 index 000000000..e942ca470 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1701.tolk @@ -0,0 +1,12 @@ +// try to cheat +fun moddiv2(x: int, y: int): (int, int) builtin; +// but actually, it remains unknown symbol + +fun main() { + moddiv2(1, 2); +} + +/** +@compilation_should_fail +@stderr undefined symbol `moddiv2` +*/ diff --git a/tolk-tester/tests/invalid-declaration/err-1764.tolk b/tolk-tester/tests/invalid-declaration/err-1764.tolk new file mode 100644 index 000000000..6afaa1849 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1764.tolk @@ -0,0 +1,9 @@ + +fun increment(mutate x: int = 0) { + x += 1; +} + +/** +@compilation_should_fail +@stderr `mutate` parameter can't have a default value + */ diff --git a/tolk-tester/tests/invalid-declaration-11.tolk b/tolk-tester/tests/invalid-declaration/err-1790.tolk similarity index 89% rename from tolk-tester/tests/invalid-declaration-11.tolk rename to tolk-tester/tests/invalid-declaration/err-1790.tolk index 75ebb450b..4a845db8d 100644 --- a/tolk-tester/tests/invalid-declaration-11.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1790.tolk @@ -1,7 +1,7 @@ // this function is declared incorrectly, // since it should return 2 values onto a stack (1 for returned slice, 1 for mutated int) // but contains not 2 numbers in asm ret_order -fun loadAddress2(mutate self: int): slice +fun int.loadAddress2(mutate self): slice asm( -> 1 0 2) "LDMSGADDR"; fun main(){} diff --git a/tolk-tester/tests/invalid-declaration/err-1794.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk new file mode 100644 index 000000000..3505572cf --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1794.tolk @@ -0,0 +1,8 @@ +get fun int.seqno(self) { + return 0; +} + +/** +@compilation_should_fail +@stderr invalid declaration of a get method + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1795.tolk b/tolk-tester/tests/invalid-declaration/err-1795.tolk new file mode 100644 index 000000000..67b062cbe --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1795.tolk @@ -0,0 +1,13 @@ +@overflow1023_policy("ignorje") +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +/** +@compilation_should_fail +@stderr incorrect value for @overflow1023_policy + */ diff --git a/tolk-tester/tests/invalid-declaration-9.tolk b/tolk-tester/tests/invalid-declaration/err-1805.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-9.tolk rename to tolk-tester/tests/invalid-declaration/err-1805.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1808.tolk b/tolk-tester/tests/invalid-declaration/err-1808.tolk new file mode 100644 index 000000000..a7faa6ff3 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1808.tolk @@ -0,0 +1,6 @@ +const C = match (10) { 10 => 0, else => -1 }; + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-declaration/err-1842.tolk similarity index 50% rename from tolk-tester/tests/invalid-cyclic-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1842.tolk index c46b1640e..0f259fa68 100644 --- a/tolk-tester/tests/invalid-cyclic-1.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1842.tolk @@ -3,6 +3,5 @@ const TWO = ONE + 1; /** @compilation_should_fail -@stderr const ONE -@stderr undefined symbol `TWO` +@stderr const `TWO` appears, directly or indirectly, in its own initializer */ diff --git a/tolk-tester/tests/invalid-declaration/err-1876.tolk b/tolk-tester/tests/invalid-declaration/err-1876.tolk new file mode 100644 index 000000000..281a4af7d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1876.tolk @@ -0,0 +1,11 @@ +struct Pair { + first: T1; + second: T2; + first: int; +} + +/** +@compilation_should_fail +@stderr redeclaration of field `first` +@stderr first: int + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1906.tolk b/tolk-tester/tests/invalid-declaration/err-1906.tolk new file mode 100644 index 000000000..2b7e458cf --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1906.tolk @@ -0,0 +1,9 @@ + +fun onBouncedMessage(in: InMessageBounced) { + return in.valueExtra; +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should return `void` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1907.tolk b/tolk-tester/tests/invalid-declaration/err-1907.tolk new file mode 100644 index 000000000..945a8bf48 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1907.tolk @@ -0,0 +1,8 @@ + +fun onBouncedMessage() { +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should have one parameter `InMessageBounced` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1908.tolk b/tolk-tester/tests/invalid-declaration/err-1908.tolk new file mode 100644 index 000000000..390441813 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1908.tolk @@ -0,0 +1,12 @@ + +fun handle(in: InMessage) { +} + +fun onInternalMessage(in: InMessage) { + return handle(in); +} + +/** +@compilation_should_fail +@stderr using `in` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1912.tolk b/tolk-tester/tests/invalid-declaration/err-1912.tolk new file mode 100644 index 000000000..2674580ca --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1912.tolk @@ -0,0 +1,9 @@ +struct Point { x: int, y: int } + +fun Point.create() {} + +/** +@compilation_should_fail +@stderr type `Point` is not generic +@stderr fun Point + */ diff --git a/tolk-tester/tests/invalid-redefinition-2.tolk b/tolk-tester/tests/invalid-declaration/err-1918.tolk similarity index 78% rename from tolk-tester/tests/invalid-redefinition-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1918.tolk index 3a300dc2e..8fec98925 100644 --- a/tolk-tester/tests/invalid-redefinition-2.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1918.tolk @@ -8,5 +8,5 @@ fun hello(): int { @compilation_should_fail @stderr fun hello() @stderr redefinition of symbol, previous was at -@stderr invalid-redefinition-2.tolk:1:1 +@stderr err-1918.tolk:1 */ diff --git a/tolk-tester/tests/invalid-get-method-1.tolk b/tolk-tester/tests/invalid-declaration/err-1940.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1940.tolk diff --git a/tolk-tester/tests/invalid-declaration/err-1973.tolk b/tolk-tester/tests/invalid-declaration/err-1973.tolk new file mode 100644 index 000000000..d00819e4b --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1973.tolk @@ -0,0 +1,7 @@ +fun int.nothing(zero: int, self: slice) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-generics-12.tolk b/tolk-tester/tests/invalid-generics-12.tolk deleted file mode 100644 index 62a6f5da4..000000000 --- a/tolk-tester/tests/invalid-generics-12.tolk +++ /dev/null @@ -1,15 +0,0 @@ -fun getTwo(): X { return 2; } - -fun cantDeduceNonArgumentGeneric() { - var t1: [int] = [0]; - t1.0 = getTwo(); // ok - var t2 = createEmptyTuple(); - t2.tuplePush(0); - t2.0 = getTwo(); // error, can't decude X -} - -/** -@compilation_should_fail -@stderr can not deduce X for generic function `getTwo` -@stderr t2.0 = getTwo(); - */ diff --git a/tolk-tester/tests/invalid-generics-14.tolk b/tolk-tester/tests/invalid-generics-14.tolk deleted file mode 100644 index eb3adc921..000000000 --- a/tolk-tester/tests/invalid-generics-14.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun eq(v: X) {} - -fun cantDeduceWhenNotInferred() { - // at type inferring (before type checking) they are unknown - var (x, y) = 2; - - eq(x as int); // ok (since execution doesn't reach type checking) - eq(x); // ok (since execution doesn't reach type checking) - eq(x); -} - -/** -@compilation_should_fail -@stderr in function `cantDeduceWhenNotInferred` -@stderr can not deduce X for generic function `eq` -@stderr eq(x); - */ diff --git a/tolk-tester/tests/invalid-generics-2.tolk b/tolk-tester/tests/invalid-generics-2.tolk deleted file mode 100644 index 155944338..000000000 --- a/tolk-tester/tests/invalid-generics-2.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun f(v: int, x: T) {} - -fun failCantDeduceWithPlainNull() { - return f(0, null); -} - -/** -@compilation_should_fail -@stderr can not deduce T for generic function `f` - */ diff --git a/tolk-tester/tests/invalid-generics-6.tolk b/tolk-tester/tests/invalid-generics-6.tolk deleted file mode 100644 index 73e6403fd..000000000 --- a/tolk-tester/tests/invalid-generics-6.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun eq(t: X) { return t; } - -fun failUsingGenericFunctionPartially() { - var cb = createEmptyTuple().eq().eq().tuplePush; -} - -/** -@compilation_should_fail -@stderr can not use a generic function `tuplePush` as non-call - */ diff --git a/tolk-tester/tests/invalid-mutate-10.tolk b/tolk-tester/tests/invalid-mutate-10.tolk deleted file mode 100644 index 8cd37c517..000000000 --- a/tolk-tester/tests/invalid-mutate-10.tolk +++ /dev/null @@ -1,16 +0,0 @@ -fun increment(mutate x: int) { - x = x + 1; -} - -fun cantCallMutatingAsAMember() { - var x = 0; - x.increment(); - return x; -} - -/** -@compilation_should_fail -@stderr function `increment` mutates parameter `x` -@stderr consider calling `increment(mutate x)`, not `x.increment`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-mutate-14.tolk b/tolk-tester/tests/invalid-mutate-14.tolk deleted file mode 100644 index 2ba645d13..000000000 --- a/tolk-tester/tests/invalid-mutate-14.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun main(cs: slice) { - return loadInt(cs, 32); -} - -/** -@compilation_should_fail -@stderr `loadInt` is a mutating method; consider calling `cs.loadInt()`, not `loadInt(cs)` - */ diff --git a/tolk-tester/tests/invalid-mutate-15.tolk b/tolk-tester/tests/invalid-mutate-15.tolk deleted file mode 100644 index f6874fb8c..000000000 --- a/tolk-tester/tests/invalid-mutate-15.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun asdf(mutate cs: slice) {} - -fun main(cs: slice) { - cs.asdf(); -} - -/** -@compilation_should_fail -@stderr function `asdf` mutates parameter `cs` -@stderr consider calling `asdf(mutate cs)`, not `cs.asdf`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-mutate-17.tolk b/tolk-tester/tests/invalid-mutate-17.tolk deleted file mode 100644 index 9327f07d8..000000000 --- a/tolk-tester/tests/invalid-mutate-17.tolk +++ /dev/null @@ -1,13 +0,0 @@ -@pure -fun tupleMut(mutate self: tuple): int - asm "TLEN"; - -fun main() { - var t = createEmptyTuple(); - return [[t.tupleMut]]; -} - -/** -@compilation_should_fail -@stderr saving `tupleMut` into a variable is impossible, since it has `mutate` parameters - */ diff --git a/tolk-tester/tests/invalid-mutate-19.tolk b/tolk-tester/tests/invalid-mutate-19.tolk deleted file mode 100644 index bb8cde058..000000000 --- a/tolk-tester/tests/invalid-mutate-19.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun getNullableTuple(): tuple? { return createEmptyTuple(); } - -fun cantUseLValueUnwrappedNotNull() { - tuplePush(mutate getNullableTuple()!, 1); -} - -/** -@compilation_should_fail -@stderr function call can not be used as lvalue - */ diff --git a/tolk-tester/tests/invalid-no-import-1.tolk b/tolk-tester/tests/invalid-no-import-1.tolk deleted file mode 100644 index 89f879a36..000000000 --- a/tolk-tester/tests/invalid-no-import-1.tolk +++ /dev/null @@ -1,8 +0,0 @@ -import "imports/some-math.tolk"; -import "imports/invalid-no-import.tolk"; - -/** -@compilation_should_fail -@stderr imports/invalid-no-import.tolk:2:13 -@stderr Using a non-imported symbol `someAdd` - */ diff --git a/tolk-tester/tests/invalid-no-import-2.tolk b/tolk-tester/tests/invalid-no-import-2.tolk deleted file mode 100644 index d78346b90..000000000 --- a/tolk-tester/tests/invalid-no-import-2.tolk +++ /dev/null @@ -1,9 +0,0 @@ -import "@stdlib/tvm-dicts" -import "imports/use-dicts-err.tolk" - -/** -@compilation_should_fail -@stderr imports/use-dicts-err.tolk:2:22 -@stderr Using a non-imported symbol `createEmptyDict` -@stderr Forgot to import "@stdlib/tvm-dicts"? - */ diff --git a/tolk-tester/tests/invalid-self-6.tolk b/tolk-tester/tests/invalid-self-6.tolk deleted file mode 100644 index 588c70ab2..000000000 --- a/tolk-tester/tests/invalid-self-6.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun increment(x: int, self: int): int { - return x + self; -} - -/** -@compilation_should_fail -@stderr `self` can only be the first parameter - */ diff --git a/tolk-tester/tests/invalid-call-5.tolk b/tolk-tester/tests/invalid-semantics/err-4002.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4002.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4057.tolk b/tolk-tester/tests/invalid-semantics/err-4057.tolk new file mode 100644 index 000000000..a5de199dc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4057.tolk @@ -0,0 +1,17 @@ +fun int16.plus1(self) { return self + 100; } +fun int8.plus1(self) { return self + 1; } + +fun main() { + (10 as int16).plus1(); // ok + (10 as int8).plus1(); // ok + + 10.plus1(); +} + +/** +@compilation_should_fail +@stderr call to method `plus1` for type `int` is ambiguous +@stderr candidate function: `int16.plus1` +@stderr candidate function: `int8.plus1` +@stderr 10.plus1 + */ diff --git a/tolk-tester/tests/invalid-mutate-5.tolk b/tolk-tester/tests/invalid-semantics/err-4061.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4061.tolk diff --git a/tolk-tester/tests/invalid-pure-1.tolk b/tolk-tester/tests/invalid-semantics/err-4064.tolk similarity index 68% rename from tolk-tester/tests/invalid-pure-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4064.tolk index 4f0e9142a..e2336ab3b 100644 --- a/tolk-tester/tests/invalid-pure-1.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4064.tolk @@ -12,9 +12,6 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -return f_impure(); -""" +@stderr an impure operation in a pure function +@stderr in function `f_pure` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4080.tolk b/tolk-tester/tests/invalid-semantics/err-4080.tolk new file mode 100644 index 000000000..df569995f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4080.tolk @@ -0,0 +1,13 @@ +fun int.methodWith1Param(self, param: int) { + +} + +fun main() { + val x = 10; + x.methodWith1Param(2, "asdf"); +} + +/** +@compilation_should_fail +@stderr too many arguments in call to `int.methodWith1Param`, expected 1, have 2 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4081.tolk b/tolk-tester/tests/invalid-semantics/err-4081.tolk new file mode 100644 index 000000000..4442cf2d5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4081.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + Wrapper => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: duplicated `Wrapper` +@stderr WrapperAlias + */ diff --git a/tolk-tester/tests/invalid-mutate-16.tolk b/tolk-tester/tests/invalid-semantics/err-4083.tolk similarity index 80% rename from tolk-tester/tests/invalid-mutate-16.tolk rename to tolk-tester/tests/invalid-semantics/err-4083.tolk index 9da6e2534..aac7417bd 100644 --- a/tolk-tester/tests/invalid-mutate-16.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4083.tolk @@ -1,6 +1,6 @@ fun cantCallMutatingFunctionWithAssignmentLValue() { var t: tuple = createEmptyTuple(); - (t = createEmptyTuple()).tuplePush(1); + (t = createEmptyTuple()).push(1); } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4085.tolk b/tolk-tester/tests/invalid-semantics/err-4085.tolk new file mode 100644 index 000000000..11c403908 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4085.tolk @@ -0,0 +1,22 @@ +struct A { + value: int; +} + +struct B { + data: slice; +} + +fun main() { + var r1: A | B = A { value: 10 }; + var r2: A | null = { value: 10 }; + var r3: A | int | builder = { value: 10 }; + + var r4: A | B = { value: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr use either `var v: StructName = { ... }` or `var v = StructName { ... }` +@stderr { value: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4104.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk new file mode 100644 index 000000000..3ca7ef1bc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4104.tolk @@ -0,0 +1,20 @@ +struct Container { + item: T; +} + +fun Container.createFrom(item: U): Container { + return { item }; +} + +fun cantUseAsNonCall() { + Container.createFrom; // ok + Container.createFrom>; // ok + + Container.createFrom; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `Container.createFrom` as non-call +@stderr Container.createFrom; + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4115.tolk b/tolk-tester/tests/invalid-semantics/err-4115.tolk new file mode 100644 index 000000000..26bcf0c1f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4115.tolk @@ -0,0 +1,14 @@ +fun (T|int).equals(self, v: U) { return self == v; } +fun T?.equals(self, v: U) { return self == v; } + +fun main() { + (10 as int?).equals(20); +} + +/** +@compilation_should_fail +@stderr error: call to method `equals` for type `int?` is ambiguous +@stderr candidate function: `(T|int).equals` with T=`null` +@stderr candidate function: `T?.equals` with T=`int` +@stderr (10 as int?).equals(20); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4119.tolk b/tolk-tester/tests/invalid-semantics/err-4119.tolk new file mode 100644 index 000000000..bb6839315 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4119.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +type WrapperAlias = Wrapper; + +fun main() { + var w1 = WrapperAlias { value: 10 } as WrapperAlias | int | Wrapper; + w1 is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr w1 is Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4127.tolk b/tolk-tester/tests/invalid-semantics/err-4127.tolk new file mode 100644 index 000000000..cae958ab6 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4127.tolk @@ -0,0 +1,7 @@ +const ttt = 1; +const asdf = ttt; + +/** +@compilation_should_fail +@stderr type arguments not expected here + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4150.tolk b/tolk-tester/tests/invalid-semantics/err-4150.tolk new file mode 100644 index 000000000..d54257872 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4150.tolk @@ -0,0 +1,13 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTensor() { + val t = (0, 1); + increment(mutate ((t!)).0); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `t` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4188.tolk b/tolk-tester/tests/invalid-semantics/err-4188.tolk new file mode 100644 index 000000000..c908ed107 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4188.tolk @@ -0,0 +1,13 @@ +fun recursive1(): int { + return recursive2(); +} + +fun recursive2(): int { + __expect_inline(true); + return recursive1(); +} + +/** +@compilation_should_fail +@stderr __expect_inline failed + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4190.tolk b/tolk-tester/tests/invalid-semantics/err-4190.tolk new file mode 100644 index 000000000..97497bd62 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4190.tolk @@ -0,0 +1,10 @@ +struct A { + ok: coins = ton("0.05"), + err: int = blockchain.now(), +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr err: int + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4193.tolk b/tolk-tester/tests/invalid-semantics/err-4193.tolk new file mode 100644 index 000000000..6e7198c61 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4193.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; +} + +fun cantInstantiateNonGenericStruct() { + var u = User { id: 3 }; +} + +/** +@compilation_should_fail +@stderr type `User` is not generic +@stderr User + */ diff --git a/tolk-tester/tests/invalid-mutate-18.tolk b/tolk-tester/tests/invalid-semantics/err-4195.tolk similarity index 75% rename from tolk-tester/tests/invalid-mutate-18.tolk rename to tolk-tester/tests/invalid-semantics/err-4195.tolk index bb8cde058..a4170ffdd 100644 --- a/tolk-tester/tests/invalid-mutate-18.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4195.tolk @@ -1,5 +1,9 @@ fun getNullableTuple(): tuple? { return createEmptyTuple(); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + fun cantUseLValueUnwrappedNotNull() { tuplePush(mutate getNullableTuple()!, 1); } diff --git a/tolk-tester/tests/invalid-semantics/err-4208.tolk b/tolk-tester/tests/invalid-semantics/err-4208.tolk new file mode 100644 index 000000000..c40dda98c --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4208.tolk @@ -0,0 +1,6 @@ +const flyp = 9999999.77777777777777; + +/** +@compilation_should_fail +@stderr invalid numeric index + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4218.tolk b/tolk-tester/tests/invalid-semantics/err-4218.tolk new file mode 100644 index 000000000..baab5a2a3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4218.tolk @@ -0,0 +1,14 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main() { + var w3: nested = {}; +} + +/** +@compilation_should_fail +@stderr field `value` missed in initialization of struct `Wrapper>>` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4256.tolk b/tolk-tester/tests/invalid-semantics/err-4256.tolk new file mode 100644 index 000000000..6b29b8aab --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4256.tolk @@ -0,0 +1,18 @@ +type MInt = int; +type IntOrSlice = MInt|slice; + +fun (int | slice).method(self) {} +fun (slice | MInt).method(self) {} +fun IntOrSlice.method(self) {} + +fun main(p: int|slice) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int | slice` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `(slice|MInt).method` +@stderr candidate function: `IntOrSlice.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4258.tolk b/tolk-tester/tests/invalid-semantics/err-4258.tolk new file mode 100644 index 000000000..d60d1adf3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4258.tolk @@ -0,0 +1,13 @@ +fun f(x: int): int { + return match(x) { + -1 => -1, + f(0) => 0, + else => 1, + }; +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr f(0) => 0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4260.tolk b/tolk-tester/tests/invalid-semantics/err-4260.tolk new file mode 100644 index 000000000..9c9a73b31 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4260.tolk @@ -0,0 +1,15 @@ +struct User { + id: int; +} + +type UserAlias = User; + +fun cantInstantiateNonGenericStruct() { + var u = UserAlias { id: 3 }; +} + +/** +@compilation_should_fail +@stderr type `UserAlias` is not generic +@stderr UserAlias + */ diff --git a/tolk-tester/tests/invalid-generics-10.tolk b/tolk-tester/tests/invalid-semantics/err-4266.tolk similarity index 57% rename from tolk-tester/tests/invalid-generics-10.tolk rename to tolk-tester/tests/invalid-semantics/err-4266.tolk index c7f72bf4d..6d6ec061b 100644 --- a/tolk-tester/tests/invalid-generics-10.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4266.tolk @@ -1,9 +1,9 @@ fun invalidReferencingGenericMethodWithoutGeneric() { var t = createEmptyTuple(); - var cb = t.tupleLast; + var cb = t.last; } /** @compilation_should_fail -@stderr can not use a generic function `tupleLast` as non-call +@stderr can not use a generic function `tuple.last` as non-call */ diff --git a/tolk-tester/tests/invalid-semantics/err-4270.tolk b/tolk-tester/tests/invalid-semantics/err-4270.tolk new file mode 100644 index 000000000..455c16345 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4270.tolk @@ -0,0 +1,8 @@ +struct A { + tens: (int, int) = (1, 2 + blockchain.now()), +} + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4291.tolk b/tolk-tester/tests/invalid-semantics/err-4291.tolk new file mode 100644 index 000000000..a7fadaa2d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4291.tolk @@ -0,0 +1,8 @@ +struct T1 { field: T1; } +struct CollisionName { item: T1; } + +/** +@compilation_should_fail +@stderr unknown type name `T1` +@stderr item: T1 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4301.tolk b/tolk-tester/tests/invalid-semantics/err-4301.tolk new file mode 100644 index 000000000..800495da3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4301.tolk @@ -0,0 +1,15 @@ +fun main() { + var i: int; + match (random.uint256()) { + 0 => { i = 0; debug.print(i) }, + 1 => { if (true) { i = 1 } }, + 2 => throw i = 2, + } + if (i == 0) {} +} + +/** +@compilation_should_fail +@stderr using variable `i` before it's definitely assigned +@stderr if (i == 0) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4304.tolk b/tolk-tester/tests/invalid-semantics/err-4304.tolk new file mode 100644 index 000000000..032586aaa --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4304.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + var value = MInt; +} + +/** +@compilation_should_fail +@stderr `MInt` only refers to a type, but is being used as a value here +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4309.tolk b/tolk-tester/tests/invalid-semantics/err-4309.tolk new file mode 100644 index 000000000..3fa24df91 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4309.tolk @@ -0,0 +1,32 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun main() { + var i: int; + if (true) { + i = 10; + } + increment(mutate i); // ok + + var j: int8 | int16; + if (random.uint256()) { + (j, _) = (10 as int8, i); + } else { + return; + } + match (j) { // ok + int8 => increment(mutate i), + int16 => increment(mutate j), + } + + var k: int; + { + increment(mutate k); + } +} + +/** +@compilation_should_fail +@stderr using variable `k` before it's definitely assigned + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4319.tolk b/tolk-tester/tests/invalid-semantics/err-4319.tolk new file mode 100644 index 000000000..f1438ea33 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4319.tolk @@ -0,0 +1,17 @@ +type MInt = int; + +fun (int | slice).method(self) {} +fun (MInt|null).method(self) {} +fun int8?.method(self) {} + +fun main(p: int8) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int8` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `MInt?.method` +@stderr candidate function: `int8?.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4320.tolk b/tolk-tester/tests/invalid-semantics/err-4320.tolk new file mode 100644 index 000000000..f57df590e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4320.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int; + y: int; +} + +fun main() { + return sizeof(Point); +} + +/** +@compilation_should_fail +@stderr `Point` only refers to a type, but is being used as a value here + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4355.tolk b/tolk-tester/tests/invalid-semantics/err-4355.tolk new file mode 100644 index 000000000..bb648dfa4 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4355.tolk @@ -0,0 +1,21 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun cantModifyValDeclaredInMatch() { + match (var a = (0 + 10 + 90) as int | slice) { + int => increment(mutate a), + slice => {}, + }; + match (val b = (0 + 10 + 90) as int | slice) { + int => increment(mutate b), + slice => {}, + }; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `b` +@stderr in function `cantModifyValDeclaredInMatch` +@stderr increment(mutate b) + */ diff --git a/tolk-tester/tests/invalid-mutate-7.tolk b/tolk-tester/tests/invalid-semantics/err-4377.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-7.tolk rename to tolk-tester/tests/invalid-semantics/err-4377.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4378.tolk b/tolk-tester/tests/invalid-semantics/err-4378.tolk new file mode 100644 index 000000000..7f1225670 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + field: T; +} + +fun f(w: Wrapper) { +} + +/** +@compilation_should_fail +@type `Wrapper` is generic, you should provide type arguments + */ diff --git a/tolk-tester/tests/invalid-typing-13.tolk b/tolk-tester/tests/invalid-semantics/err-4381.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-13.tolk rename to tolk-tester/tests/invalid-semantics/err-4381.tolk diff --git a/tolk-tester/tests/invalid-pure-3.tolk b/tolk-tester/tests/invalid-semantics/err-4385.tolk similarity index 61% rename from tolk-tester/tests/invalid-pure-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4385.tolk index 31d4f0213..767f524f6 100644 --- a/tolk-tester/tests/invalid-pure-3.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4385.tolk @@ -1,6 +1,6 @@ @pure fun validate_input(input: cell): (int, int) { - var (x, y, z, correct) = calculateCellSize(input, 10); + var (x, y, z, correct) = input.calculateSize(10); assert(correct) throw 102; return (x, y); } @@ -16,9 +16,7 @@ fun main() {} /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -assert(correct) -""" +@stderr an impure operation in a pure function +@stderr in function `validate_input` +@stderr assert(correct) */ diff --git a/tolk-tester/tests/invalid-semantics/err-4387.tolk b/tolk-tester/tests/invalid-semantics/err-4387.tolk new file mode 100644 index 000000000..0de2871ac --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4387.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main(d: nested) { + var c1 = d; + + var c2 = Wrapper>>; +} + +/** +@compilation_should_fail +@stderr `Wrapper` only refers to a type, but is being used as a value here +@stderr c2 = Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4391.tolk b/tolk-tester/tests/invalid-semantics/err-4391.tolk new file mode 100644 index 000000000..80baf9b83 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4391.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; + name: slice; +} + +fun main() { + var u: User = { id: 1 }; +} + +/** +@compilation_should_fail +@stderr field `name` missed in initialization of struct `User` + */ diff --git a/tolk-tester/tests/invalid-pure-2.tolk b/tolk-tester/tests/invalid-semantics/err-4413.tolk similarity index 67% rename from tolk-tester/tests/invalid-pure-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4413.tolk index 213206834..6ee576dcc 100644 --- a/tolk-tester/tests/invalid-pure-2.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4413.tolk @@ -15,9 +15,7 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -g = g + 1; -""" -*/ +@stderr an impure operation in a pure function +@stderr in function `f_pure` +@stderr g = g + 1; + */ diff --git a/tolk-tester/tests/invalid-generics-5.tolk b/tolk-tester/tests/invalid-semantics/err-4416.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4416.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4429.tolk b/tolk-tester/tests/invalid-semantics/err-4429.tolk new file mode 100644 index 000000000..028ae1078 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4429.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; + +fun main(w: int | Wrapper | Wrapper) { + w is Wrapper; + w is Wrapper; + w is Wrapper; + w is Wrapper; + + w is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4440.tolk b/tolk-tester/tests/invalid-semantics/err-4440.tolk new file mode 100644 index 000000000..9c59f2c50 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4440.tolk @@ -0,0 +1,21 @@ +struct NumberFields { + `-1`: int; + `0`: int; + `100500`: int; +} + +fun main() { + var b: NumberFields = { `-1`: -1, `0`: 0, `100500`: 100500 }; + b.0; // ok + b.`0`; // ok + b.`-1`; // ok + b.`100500`; // ok + b.100500; // ok + b.123; +} + +/** +@compilation_should_fail +@stderr field `123` doesn't exist in type `NumberFields` +@stderr b.123 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4460.tolk b/tolk-tester/tests/invalid-semantics/err-4460.tolk new file mode 100644 index 000000000..1cae8c7a9 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4460.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = lazy Point.fromSlice(stringHexToSlice("0A14")); + p.asdf; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `Point` + */ diff --git a/tolk-tester/tests/invalid-mutate-11.tolk b/tolk-tester/tests/invalid-semantics/err-4473.tolk similarity index 79% rename from tolk-tester/tests/invalid-mutate-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4473.tolk index dfc69851c..9d4ad1cc0 100644 --- a/tolk-tester/tests/invalid-mutate-11.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4473.tolk @@ -1,4 +1,4 @@ -fun load32(self: slice): int { +fun slice.load32(self): int { return self.loadUint(32); } diff --git a/tolk-tester/tests/invalid-semantics/err-4480.tolk b/tolk-tester/tests/invalid-semantics/err-4480.tolk new file mode 100644 index 000000000..a0f0da32d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4480.tolk @@ -0,0 +1,10 @@ +fun main() { + Point2.create(); +} + +/** +@compilation_should_fail +@stderr unknown type name `Point2` +@stderr in function `main` +@stderr Point2.create() + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4493.tolk b/tolk-tester/tests/invalid-semantics/err-4493.tolk new file mode 100644 index 000000000..8a386f8c3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4493.tolk @@ -0,0 +1,13 @@ +struct Wrapper { + value: T; +} + +fun my() { + var w: Wrapper = Wrapper { value: 30 }; +} + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr var w: Wrapper = + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4509.tolk b/tolk-tester/tests/invalid-semantics/err-4509.tolk new file mode 100644 index 000000000..d168ddb96 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4509.tolk @@ -0,0 +1,14 @@ +fun f(a: int = 0, b: int) { + +} + +fun main() { + f(1, 2); // ok + return f(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `f`, expected 2, have 0 +@stderr f(); + */ diff --git a/tolk-tester/tests/invalid-generics-11.tolk b/tolk-tester/tests/invalid-semantics/err-4510.tolk similarity index 71% rename from tolk-tester/tests/invalid-generics-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4510.tolk index a399bc917..82aaaaccf 100644 --- a/tolk-tester/tests/invalid-generics-11.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4510.tolk @@ -7,5 +7,5 @@ fun main() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4512.tolk b/tolk-tester/tests/invalid-semantics/err-4512.tolk new file mode 100644 index 000000000..77c18cb89 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4512.tolk @@ -0,0 +1,8 @@ +fun f() { + var n: int = 10; +} + +/** +@compilation_should_fail +@stderr type `int` is not generic + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4517.tolk b/tolk-tester/tests/invalid-semantics/err-4517.tolk new file mode 100644 index 000000000..5497ebb41 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4517.tolk @@ -0,0 +1,14 @@ +global g: int; + +fun f(x: int): int { + match(x) { + -1 => -1, + g => 0, + }; + return 10; +} + +/** +@compilation_should_fail +@stderr unknown type name `g` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4519.tolk b/tolk-tester/tests/invalid-semantics/err-4519.tolk new file mode 100644 index 000000000..a76d20a96 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4519.tolk @@ -0,0 +1,14 @@ +struct value { id: int; } + +fun main() { + var value = 100; + var value2 = 200; + var obj1 = value { id: 100 }; // "value" refs to a struct, it's parsed as a type + var obj2 = value2 { id: 200 }; // no struct "value2" +} + +/** +@compilation_should_fail +@stderr unknown type name `value2` +@stderr obj2 = value2 + */ diff --git a/tolk-tester/tests/invalid-call-8.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk similarity index 79% rename from tolk-tester/tests/invalid-call-8.tolk rename to tolk-tester/tests/invalid-semantics/err-4520.tolk index 199aa681a..47b1d2d55 100644 --- a/tolk-tester/tests/invalid-call-8.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4520.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr type `int` is not indexable +@stderr method `3` not found */ diff --git a/tolk-tester/tests/invalid-semantics/err-4530.tolk b/tolk-tester/tests/invalid-semantics/err-4530.tolk new file mode 100644 index 000000000..c7adb10d7 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4530.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + field: T; +} + +fun main() { + var w1: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w2: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w3: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + + var w4: int | Wrapper | null | Wrapper | (int, int) = { field: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr { field: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4542.tolk b/tolk-tester/tests/invalid-semantics/err-4542.tolk new file mode 100644 index 000000000..e1ef833e5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4542.tolk @@ -0,0 +1,13 @@ +@pure +fun tuple.tupleMut(mutate self): int + asm "TLEN"; + +fun main() { + var t = createEmptyTuple(); + return [[t.tupleMut]]; +} + +/** +@compilation_should_fail +@stderr saving `tuple.tupleMut` into a variable is impossible, since it has `mutate` parameters + */ diff --git a/tolk-tester/tests/invalid-mutate-4.tolk b/tolk-tester/tests/invalid-semantics/err-4553.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-4.tolk rename to tolk-tester/tests/invalid-semantics/err-4553.tolk diff --git a/tolk-tester/tests/invalid-syntax-3.tolk b/tolk-tester/tests/invalid-semantics/err-4571.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4571.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4572.tolk b/tolk-tester/tests/invalid-semantics/err-4572.tolk new file mode 100644 index 000000000..a62b63a0f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4572.tolk @@ -0,0 +1,9 @@ +fun checkCantMutateFieldOfImmutableTuple() { + val ks = null as [int, [int, [int]]]?; + ks!.1.1.0 = 10; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `ks` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4580.tolk b/tolk-tester/tests/invalid-semantics/err-4580.tolk new file mode 100644 index 000000000..bd414eeb6 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4580.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + item: T; +} + +fun main() { + var c = Wrapper { }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4588.tolk b/tolk-tester/tests/invalid-semantics/err-4588.tolk new file mode 100644 index 000000000..9c4ded675 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4588.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int; + y: int; +} + +fun getX(i: int) { return i; } +fun Point.getX(self) { return self.x; } + +fun main() { + var Point = 10; + Point.getX(); +} + +/** +@compilation_should_fail +@stderr method `getX` not found for type `int` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4598.tolk b/tolk-tester/tests/invalid-semantics/err-4598.tolk new file mode 100644 index 000000000..3ef699380 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4598.tolk @@ -0,0 +1,16 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +fun main(c: Container) { + c.wrap(); +} + +/** +@compilation_should_fail +@stderr method `Container.wrap` can not be called via dot +@stderr it's a static method, it does not accept `self` +@stderr c.wrap(); + */ diff --git a/tolk-tester/tests/invalid-generics-9.tolk b/tolk-tester/tests/invalid-semantics/err-4604.tolk similarity index 66% rename from tolk-tester/tests/invalid-generics-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4604.tolk index 73fd6f87c..07637d9da 100644 --- a/tolk-tester/tests/invalid-generics-9.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4604.tolk @@ -4,5 +4,5 @@ fun invalidProvidingGenericTsToNotGeneric() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4610.tolk b/tolk-tester/tests/invalid-semantics/err-4610.tolk new file mode 100644 index 000000000..2c495602d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4610.tolk @@ -0,0 +1,13 @@ +fun int.create(value: int) { + return value; +} + +fun main() { + 10.create(); +} + +/** +@compilation_should_fail +@stderr method `int.create` can not be called via dot +@stderr it's a static method, it does not accept `self` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4618.tolk b/tolk-tester/tests/invalid-semantics/err-4618.tolk new file mode 100644 index 000000000..6fb6419a8 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4618.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + onExternalMessage(""); +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr onExternalMessage is a special entrypoint, it can not be called as a regular function + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4628.tolk b/tolk-tester/tests/invalid-semantics/err-4628.tolk new file mode 100644 index 000000000..1f7debb5e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4628.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr can not deduce T for generic struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4629.tolk b/tolk-tester/tests/invalid-semantics/err-4629.tolk new file mode 100644 index 000000000..dc69a83be --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4629.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4630.tolk b/tolk-tester/tests/invalid-semantics/err-4630.tolk new file mode 100644 index 000000000..d404adbde --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4630.tolk @@ -0,0 +1,13 @@ +struct Ok { result: T; } + +type OkInt = Ok; +type OkSlice = Ok; + +fun f(w: OkInt | int | OkSlice) { + if (w !is Ok) {} +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Ok`, provide them manually + */ diff --git a/tolk-tester/tests/invalid-syntax-6.tolk b/tolk-tester/tests/invalid-semantics/err-4639.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4639.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4682.tolk b/tolk-tester/tests/invalid-semantics/err-4682.tolk new file mode 100644 index 000000000..0a042411b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4682.tolk @@ -0,0 +1,13 @@ +struct User { id: int; } + +fun absk(v: int) { return v > 0 ? v : -v; } + +fun main() { + absk(User); +} + +/** +@compilation_should_fail +@stderr `User` only refers to a type, but is being used as a value here +@stderr absk(User) +*/ diff --git a/tolk-tester/tests/invalid-mutate-12.tolk b/tolk-tester/tests/invalid-semantics/err-4705.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-12.tolk rename to tolk-tester/tests/invalid-semantics/err-4705.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4712.tolk b/tolk-tester/tests/invalid-semantics/err-4712.tolk new file mode 100644 index 000000000..2e210eb4b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4712.tolk @@ -0,0 +1,10 @@ +fun main() { + var t: tuple = createEmptyTuple(); + t.asdf = 10; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `tuple` +@stderr in function `main` +*/ diff --git a/tolk-tester/tests/invalid-mutate-1.tolk b/tolk-tester/tests/invalid-semantics/err-4728.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4728.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4739.tolk b/tolk-tester/tests/invalid-semantics/err-4739.tolk new file mode 100644 index 000000000..58fa5efe5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4739.tolk @@ -0,0 +1,12 @@ +fun T.pairWith(self, value: U) { + return [self, value]; +} + +fun main() { + 10.pairWith(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `T.pairWith`, expected 1, have 0 + */ diff --git a/tolk-tester/tests/invalid-call-3.tolk b/tolk-tester/tests/invalid-semantics/err-4751.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4751.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4790.tolk b/tolk-tester/tests/invalid-semantics/err-4790.tolk new file mode 100644 index 000000000..910d74114 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4790.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + var f = onExternalMessage; +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's a special entrypoint + */ diff --git a/tolk-tester/tests/invalid-call-11.tolk b/tolk-tester/tests/invalid-semantics/err-4804.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4804.tolk diff --git a/tolk-tester/tests/invalid-mutate-3.tolk b/tolk-tester/tests/invalid-semantics/err-4819.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4819.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4822.tolk b/tolk-tester/tests/invalid-semantics/err-4822.tolk new file mode 100644 index 000000000..85cf4073b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4822.tolk @@ -0,0 +1,23 @@ +struct Point { + x: int; + y: int; +} + +struct Coord2 { + p1: Point?; + p2: Point; +} + +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { + val c2: Coord2 = { p1, p2 }; + (c2!.p1!).x.increment(); +} + +/** +@compilation_should_fail +@modifying immutable variable `c2` + */ diff --git a/tolk-tester/tests/invalid-mutate-6.tolk b/tolk-tester/tests/invalid-semantics/err-4838.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4838.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4847.tolk b/tolk-tester/tests/invalid-semantics/err-4847.tolk new file mode 100644 index 000000000..215c245c3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4847.tolk @@ -0,0 +1,16 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun testCantCallWithoutMutate() { + var x = 0; + increment(x); +} + +/** +@compilation_should_fail +@stderr function `increment` mutates parameter `x` +@stderr you need to specify `mutate` when passing an argument, like `mutate x` +@stderr in function `testCantCallWithoutMutate` +@stderr increment(x) + */ diff --git a/tolk-tester/tests/invalid-mutate-9.tolk b/tolk-tester/tests/invalid-semantics/err-4851.tolk similarity index 85% rename from tolk-tester/tests/invalid-mutate-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4851.tolk index 3489a2889..5195da070 100644 --- a/tolk-tester/tests/invalid-mutate-9.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4851.tolk @@ -1,4 +1,4 @@ -fun increment(self: int) { +fun int.increment(self) { self = self + 1; } diff --git a/tolk-tester/tests/invalid-semantics/err-4862.tolk b/tolk-tester/tests/invalid-semantics/err-4862.tolk new file mode 100644 index 000000000..490c9d2f3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4862.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: T1; + second: T2; +} + +fun cantProvideDuplicateField() { + var p: Pair = { + first: null, + second: null, + first: 3, + }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +@stderr first: 3 + */ diff --git a/tolk-tester/tests/invalid-generics-1.tolk b/tolk-tester/tests/invalid-semantics/err-4880.tolk similarity index 64% rename from tolk-tester/tests/invalid-generics-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4880.tolk index 0bbdeee67..4564a4a81 100644 --- a/tolk-tester/tests/invalid-generics-1.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4880.tolk @@ -6,5 +6,5 @@ fun failCantDeduceWithoutArgument() { /** @compilation_should_fail -@stderr too few arguments in call to `f`, expected 2, have 1 +@stderr too few arguments in call to `f`, expected 2, have 1 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4884.tolk b/tolk-tester/tests/invalid-semantics/err-4884.tolk new file mode 100644 index 000000000..0515a4b2e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4884.tolk @@ -0,0 +1,13 @@ +fun T.copy(self) { + return self; +} + +fun main() { + int.copy(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `int.copy`, expected 1, have 0 +@stderr int.copy(); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4890.tolk b/tolk-tester/tests/invalid-semantics/err-4890.tolk new file mode 100644 index 000000000..d913652a0 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4890.tolk @@ -0,0 +1,11 @@ + +fun onInternalMessage(in: InMessage) { + in.valueCoins = 123; + return in.valueCoins; +} + +/** +@compilation_should_fail +@stderr modifying an immutable variable +@stderr hint: fields of InMessage can be used for reading only + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4907.tolk b/tolk-tester/tests/invalid-semantics/err-4907.tolk new file mode 100644 index 000000000..0674c933b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4907.tolk @@ -0,0 +1,25 @@ +struct Data { + data: T; +} + +struct Parameters { + bounce: bool, + dest: slice | Data, + body: TBody, +} + +fun f(params: Parameters) { +} + +fun main() { + f({ + dest: "sss", + body: 123, + }) +} + +/** +@compilation_should_fail +@stderr can not deduce TData for generic struct `Parameters` +@stderr dest: "sss" + */ diff --git a/tolk-tester/tests/invalid-mutate-2.tolk b/tolk-tester/tests/invalid-semantics/err-4912.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4912.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4925.tolk b/tolk-tester/tests/invalid-semantics/err-4925.tolk new file mode 100644 index 000000000..858ff6f77 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4925.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + WrapperAlias => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `WrapperAlias`, provide them manually +@stderr WrapperAlias => {} + */ diff --git a/tolk-tester/tests/invalid-call-9.tolk b/tolk-tester/tests/invalid-semantics/err-4951.tolk similarity index 65% rename from tolk-tester/tests/invalid-call-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4951.tolk index 87eb61e84..6766b2490 100644 --- a/tolk-tester/tests/invalid-call-9.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4951.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-call-2.tolk b/tolk-tester/tests/invalid-semantics/err-4956.tolk similarity index 77% rename from tolk-tester/tests/invalid-call-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4956.tolk index 5a8c9fa5d..de5309dc9 100644 --- a/tolk-tester/tests/invalid-call-2.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4956.tolk @@ -10,5 +10,5 @@ fun main() { /** @compilation_should_fail -@stderr `mutate` used for non-mutate argument +@stderr `mutate` used for non-mutate parameter */ diff --git a/tolk-tester/tests/invalid-serialization/err-7008.tolk b/tolk-tester/tests/invalid-serialization/err-7008.tolk new file mode 100644 index 000000000..9203ea228 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7008.tolk @@ -0,0 +1,16 @@ +struct Point { + x: int; + y: int; +} + +fun main(p: Point) { + p.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `Point` +@stderr because field `Point.x` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7114.tolk b/tolk-tester/tests/invalid-serialization/err-7114.tolk new file mode 100644 index 000000000..2507e81f2 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7114.tolk @@ -0,0 +1,22 @@ +type NotSerializableTensor = (int8, slice); + +struct Demo { + a: int8; + b: int8 | int16; + c: NotSerializableTensor; +} + +fun main() { + Demo.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `Demo` +@stderr because field `Demo.c` of type `NotSerializableTensor` can't be serialized +@stderr because alias `NotSerializableTensor` expands to `(int8, slice)` +@stderr because element `tensor.1` of type `slice` can't be serialized +@stderr because type `slice` can not be used for reading, it doesn't define binary width +@stderr hint: replace `slice` with `address` if it's an address, actually +@stderr hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7215.tolk b/tolk-tester/tests/invalid-serialization/err-7215.tolk new file mode 100644 index 000000000..63b46affc --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7215.tolk @@ -0,0 +1,23 @@ +type MInt = int; + +struct CantBe { + a: int8; + b: MInt?; +} + +struct Container { + item: T; +} + +fun main(s: slice) { + Container.fromSlice(s); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `Container` +@stderr because field `Container.item` of type `CantBe` can't be serialized +@stderr because field `CantBe.b` of type `MInt?` can't be serialized +@stderr because alias `MInt` expands to `int` +@stderr because type `int` is not serializable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7300.tolk b/tolk-tester/tests/invalid-serialization/err-7300.tolk new file mode 100644 index 000000000..31930736e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7300.tolk @@ -0,0 +1,15 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +fun f() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy(""); + return (o is A) ? o.a1 : o.b1; +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `o` it's used in a non-lazy manner +@stderr hint: lazy union may be used only in `match` statement + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7301.tolk b/tolk-tester/tests/invalid-serialization/err-7301.tolk new file mode 100644 index 000000000..1843bc483 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7301.tolk @@ -0,0 +1,14 @@ +type Int32Or64 = int32 | int64; + +fun f() { + val o = lazy Int32Or64.fromSlice(""); + match (o) { + int32 => return o + 1, + int64 => throw o, + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int32` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7302.tolk b/tolk-tester/tests/invalid-serialization/err-7302.tolk new file mode 100644 index 000000000..7885f6d0c --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7302.tolk @@ -0,0 +1,16 @@ +struct A{} +type StructAndNot = A | int8; + +fun f() { + var o = lazy StructAndNot.fromSlice(""); // neg5 + __expect_lazy(""); + match (o) { + A => {} + int8 => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7303.tolk b/tolk-tester/tests/invalid-serialization/err-7303.tolk new file mode 100644 index 000000000..3fe030217 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7303.tolk @@ -0,0 +1,11 @@ +type PairInt8 = (int8, int8); + +fun f() { + var i = lazy PairInt8.fromSlice(stringHexToSlice("0102")); + return (i.0, i.1); +} + +/** +@compilation_should_fail +@stderr `lazy` is applicable to structs, not to `PairInt8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7304.tolk b/tolk-tester/tests/invalid-serialization/err-7304.tolk new file mode 100644 index 000000000..4659caf80 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7304.tolk @@ -0,0 +1,22 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f(c: cell) { + // because 2 matches + var msg = lazy MsgEitherCounter.fromCell(c); // neg8 + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7305.tolk b/tolk-tester/tests/invalid-serialization/err-7305.tolk new file mode 100644 index 000000000..59fb5ba50 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7305.tolk @@ -0,0 +1,19 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because .toCell() + var msg = lazy MsgEitherCounter.fromSlice(""); // neg9 + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + return msg.toCell(); +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr lazy MsgEitherCounter + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7306.tolk b/tolk-tester/tests/invalid-serialization/err-7306.tolk new file mode 100644 index 000000000..d515c7d49 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7306.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because in a complex expression + var msg = lazy MsgEitherCounter.fromSlice(""); + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + }.toCell() +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7307.tolk b/tolk-tester/tests/invalid-serialization/err-7307.tolk new file mode 100644 index 000000000..9265cf1ce --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7307.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + var msg = lazy MsgEitherCounter.fromSlice(""); // neg10 + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + else => throw 123, // not allowed in either + } +} + +/** +@compilation_should_fail +@stderr `else` is unreachable, because this `match` has only two options (0/1 prefixes) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7308.tolk b/tolk-tester/tests/invalid-serialization/err-7308.tolk new file mode 100644 index 000000000..a46964a86 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7308.tolk @@ -0,0 +1,18 @@ +struct (0x01) A { } +struct (0x02) B { } +struct (0x03) C { } +type MyInput = A | B | C; + +fun f() { + var msg = lazy MyInput.fromSlice(""); + match (msg) { + A => {} + B => {} + else => throw 123, + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `C` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7419.tolk b/tolk-tester/tests/invalid-serialization/err-7419.tolk new file mode 100644 index 000000000..4133c80a0 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7419.tolk @@ -0,0 +1,24 @@ +struct(0x1) Ok1 {} +struct(0x2) Ok2 {} +struct(0x3) Ok3 {} +struct NoPrefix {} + +type MsgOk1 = Ok1 | Ok2 | Ok3; + +type MsgCantBe = Ok1 | Ok2 | NoPrefix | Ok3; + +fun main() { + MsgOk1.fromCell(beginCell().endCell()); + + MsgCantBe.fromCell(beginCell().endCell()); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `MsgCantBe` +@stderr because alias `MsgCantBe` expands to `Ok1 | Ok2 | NoPrefix | Ok3` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `Ok3` has opcode, but `NoPrefix` does not +@stderr hint: manually specify opcodes to all structures +@stderr err-7419.tolk:13 + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7514.tolk b/tolk-tester/tests/invalid-serialization/err-7514.tolk new file mode 100644 index 000000000..11728a6c7 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7514.tolk @@ -0,0 +1,47 @@ +struct In4 { + b: builder; +} + +struct In3 { + i: In4; +} + +struct In2 { + i: In3?; +} + +struct In1 { + i: (In2, slice); +} + +struct MaybeNothing {} +struct MaybeJust { value: T } +type Maybe = MaybeNothing | MaybeJust; + +struct CantBe { + a: address; + b: address?; + i: Maybe; +} + +fun main(c: cell) { + var d: CantBe = CantBe.fromCell(c); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `CantBe` +@stderr because field `CantBe.i` of type `Maybe` can't be serialized +@stderr because alias `Maybe` expands to `MaybeNothing | MaybeJust` +@stderr because variant #2 of type `MaybeJust` can't be serialized +@stderr because field `MaybeJust.value` of type `In1` can't be serialized +@stderr because field `In1.i` of type `(In2, slice)` can't be serialized +@stderr because element `tensor.0` of type `In2` can't be serialized +@stderr because field `In2.i` of type `In3?` can't be serialized +@stderr because field `In3.i` of type `In4` can't be serialized +@stderr because field `In4.b` of type `builder` can't be serialized +@stderr because type `builder` can not be used for reading, only for writing +@stderr hint: use `bitsN` or `RemainingBitsAndRefs` for reading +@stderr hint: using generics, you can substitute `builder` for writing and something other for reading +@stderr CantBe.fromCell(c) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7563.tolk b/tolk-tester/tests/invalid-serialization/err-7563.tolk new file mode 100644 index 000000000..9934ebb6e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7563.tolk @@ -0,0 +1,20 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +struct B { + a: Cell; +} + +fun main(b: B) { + B.fromSlice("").a.load(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7564.tolk b/tolk-tester/tests/invalid-serialization/err-7564.tolk new file mode 100644 index 000000000..36eb63d4f --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7564.tolk @@ -0,0 +1,16 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +fun main(o: A | int32) { + o.toCell(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7604.tolk b/tolk-tester/tests/invalid-serialization/err-7604.tolk new file mode 100644 index 000000000..ea6edfe7a --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7604.tolk @@ -0,0 +1,21 @@ +struct CCC { + v: int; +} + +fun main() { + var options = CreateMessageOptions { + bounce: true, + dest: createAddressNone(), + value: 0, + body: CCC { v: 10 } + }; + createMessage(options); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via createMessage() is not available for type `CCC` +@stderr because field `CCC.v` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7605.tolk b/tolk-tester/tests/invalid-serialization/err-7605.tolk new file mode 100644 index 000000000..087e56457 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7605.tolk @@ -0,0 +1,19 @@ +struct BigBodyMessage { + v1: int256; + v2: int256; + v3: int256; + a1: address; + a2: address; +} + +fun testFail(body: BigBodyMessage) { + createExternalLogMessage({ + dest: createAddressNone(), + body, + }); +} + +/** +@compilation_should_fail +@stderr struct `BigBodyMessage` can exceed 1023 bits in serialization (estimated size: 772..1302 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7683.tolk b/tolk-tester/tests/invalid-serialization/err-7683.tolk new file mode 100644 index 000000000..f82924c90 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7683.tolk @@ -0,0 +1,12 @@ +struct A { +} + +fun neg4() { + // because compiler doesn't analyze deep assignments + var (a, b) = (1, lazy A.fromSlice("")); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7684.tolk b/tolk-tester/tests/invalid-serialization/err-7684.tolk new file mode 100644 index 000000000..cfb49b3ae --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7684.tolk @@ -0,0 +1,11 @@ +struct St { +} + +fun getStorage() { + return lazy St.fromCell(contract.getData()); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7720.tolk b/tolk-tester/tests/invalid-serialization/err-7720.tolk new file mode 100644 index 000000000..ec1bc4d35 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7720.tolk @@ -0,0 +1,15 @@ +struct Point { + x: int8; + y: int8; +} + +fun neg11(s: slice) { + // mutating function can't be lazy + var p = lazy s.loadAny(); + return p.x; +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them +*/ diff --git a/tolk-tester/tests/invalid-serialization/err-7750.tolk b/tolk-tester/tests/invalid-serialization/err-7750.tolk new file mode 100644 index 000000000..fcc5329ab --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7750.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +struct WithC { + a: int32; + b: CustomInt; +} + +fun main() { + WithC.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `WithC` +@stderr because field `WithC.b` of type `CustomInt` can't be serialized +@stderr because type `CustomInt` defines a custom pack function, but does not define unpack +@stderr hint: declare unpacker like this: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice): CustomInt + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7751.tolk b/tolk-tester/tests/invalid-serialization/err-7751.tolk new file mode 100644 index 000000000..41ddd142e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7751.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.packToBuilder()` is declared incorrectly +@stderr hint: it must accept 2 parameters and return nothing: +@stderr fun CustomInt.packToBuilder(self, mutate b: builder) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7752.tolk b/tolk-tester/tests/invalid-serialization/err-7752.tolk new file mode 100644 index 000000000..b8dc0b9ec --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7752.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(self, mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7753.tolk b/tolk-tester/tests/invalid-serialization/err-7753.tolk new file mode 100644 index 000000000..6e30b474b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7753.tolk @@ -0,0 +1,19 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + return (1, 2, 3) +} + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7754.tolk b/tolk-tester/tests/invalid-serialization/err-7754.tolk new file mode 100644 index 000000000..21127dd47 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7754.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + if (s.remainingBitsCount() > 10) { + return 123; + } + return 600; +} + +fun main() { + var s = ""; + s.loadAny(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via loadAny() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7800.tolk b/tolk-tester/tests/invalid-serialization/err-7800.tolk new file mode 100644 index 000000000..1df8e5f5c --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7800.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int32; + y: int32; +} + +fun loadPoint(s: slice): Point { + return Point.fromSlice(s); +} + +fun main() { + var p = lazy loadPoint(stringHexToSlice("0000000100000002")); +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7810.tolk b/tolk-tester/tests/invalid-serialization/err-7810.tolk new file mode 100644 index 000000000..2cf0d32e5 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7810.tolk @@ -0,0 +1,20 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | int32; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | int32` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7811.tolk b/tolk-tester/tests/invalid-serialization/err-7811.tolk new file mode 100644 index 000000000..5dbcab7e6 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7811.tolk @@ -0,0 +1,19 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | null; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | null` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7812.tolk b/tolk-tester/tests/invalid-serialization/err-7812.tolk new file mode 100644 index 000000000..0b6995633 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7812.tolk @@ -0,0 +1,20 @@ +struct(0b000) B {} +struct C {} + +struct A { + multiple: (int32, B | C); +} + +fun main() { + var a: A = { multiple: (5, B{}) }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `(int32, B | C)` can't be serialized +@stderr because element `tensor.1` of type `B | C` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `B` has opcode, but `C` does not + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7813.tolk b/tolk-tester/tests/invalid-serialization/err-7813.tolk new file mode 100644 index 000000000..720678cff --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7813.tolk @@ -0,0 +1,13 @@ +struct (0x0F) A {} + +fun f(x: int32 | A) { + x.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `int32 | A` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `A` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7911.tolk b/tolk-tester/tests/invalid-serialization/err-7911.tolk new file mode 100644 index 000000000..ab045dd40 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7911.tolk @@ -0,0 +1,22 @@ +struct ExtraData { + owner: address; + lastTime: int; +} + +struct Storage { + more: Cell; +} + +fun main() { + Storage.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via fromSlice() is not available for type `Storage` +@stderr because field `Storage.more` of type `Cell` can't be serialized +@stderr because type `ExtraData` can't be serialized +@stderr because field `ExtraData.lastTime` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7923.tolk b/tolk-tester/tests/invalid-serialization/err-7923.tolk new file mode 100644 index 000000000..8b809560d --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7923.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = Point.fromSlice(stringHexToSlice("0102")); + p.forceLoadLazyObject(); +} + +/** +@compilation_should_fail +@stderr this method is applicable to lazy variables only + */ diff --git a/tolk-tester/tests/invalid-mutate-8.tolk b/tolk-tester/tests/invalid-symbol/err-2055.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-8.tolk rename to tolk-tester/tests/invalid-symbol/err-2055.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2098.tolk b/tolk-tester/tests/invalid-symbol/err-2098.tolk new file mode 100644 index 000000000..5c2b952bb --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2098.tolk @@ -0,0 +1,9 @@ +struct A { + a: int = 1 << CCC; +} + +/** +@compilation_should_fail +@stderr undefined symbol `CCC` +@stderr 1 << CCC + */ diff --git a/tolk-tester/tests/invalid-call-7.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk similarity index 77% rename from tolk-tester/tests/invalid-call-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2188.tolk index cf8c788c1..4d0c66d3d 100644 --- a/tolk-tester/tests/invalid-call-7.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2188.tolk @@ -9,6 +9,6 @@ fun main() { /** @compilation_should_fail -@stderr non-existing method `storeUnexisting` of type `builder` +@stderr method `storeUnexisting` not found @stderr .storeUnexisting() */ diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk new file mode 100644 index 000000000..c3d47a672 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2205.tolk @@ -0,0 +1,8 @@ +import "../imports/some-math.tolk"; +import "../imports/invalid-no-import.tolk"; + +/** +@compilation_should_fail +@stderr tests/imports/invalid-no-import.tolk:2:13 +@stderr Using a non-imported symbol `someAdd` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2208.tolk b/tolk-tester/tests/invalid-symbol/err-2208.tolk new file mode 100644 index 000000000..b918e0abf --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2208.tolk @@ -0,0 +1,13 @@ +fun main(x: int | slice) { + match (var a = x) { // a variable declared in `match` subject exists only within a match + int => a + 0, + slice => a.loadInt(32), + } + return a; +} + +/** +@compilation_should_fail +@stderr undefined symbol `a` +@stderr return a; + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2218.tolk b/tolk-tester/tests/invalid-symbol/err-2218.tolk new file mode 100644 index 000000000..a9a1e7887 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2218.tolk @@ -0,0 +1,13 @@ +fun demo(x: int) { + +} + +fun main() { + 10.demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found, but there is a global function named `demo` +@stderr (a function should be called `foo(arg)`, not `arg.foo()`) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2255.tolk b/tolk-tester/tests/invalid-symbol/err-2255.tolk new file mode 100644 index 000000000..134a8c485 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2255.tolk @@ -0,0 +1,8 @@ +fun testUInt258DoesntExist() { + var c = 0 as uint258; +} + +/** +@compilation_should_fail +@stderr unknown type name `uint258` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2355.tolk b/tolk-tester/tests/invalid-symbol/err-2355.tolk new file mode 100644 index 000000000..f85dc2591 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2355.tolk @@ -0,0 +1,12 @@ + +fun main() { + match (10) { + asdf => 1, + }; +} + +/** +@compilation_should_fail +@stderr unknown type name `asdf` +@stderr asdf => 1 + */ diff --git a/tolk-tester/tests/invalid-symbol-1.tolk b/tolk-tester/tests/invalid-symbol/err-2374.tolk similarity index 80% rename from tolk-tester/tests/invalid-symbol-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2374.tolk index 08a86f176..3a3ba567c 100644 --- a/tolk-tester/tests/invalid-symbol-1.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2374.tolk @@ -4,11 +4,11 @@ fun main(x: int): int { } else { var y: slice = "20"; } - debugPrint(y); + debug.print(y); } /** @compilation_should_fail -@stderr debugPrint(y); +@stderr debug.print(y); @stderr undefined symbol `y` */ diff --git a/tolk-tester/tests/invalid-symbol-2.tolk b/tolk-tester/tests/invalid-symbol/err-2463.tolk similarity index 100% rename from tolk-tester/tests/invalid-symbol-2.tolk rename to tolk-tester/tests/invalid-symbol/err-2463.tolk diff --git a/tolk-tester/tests/invalid-redefinition-6.tolk b/tolk-tester/tests/invalid-symbol/err-2501.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-6.tolk rename to tolk-tester/tests/invalid-symbol/err-2501.tolk diff --git a/tolk-tester/tests/invalid-self-7.tolk b/tolk-tester/tests/invalid-symbol/err-2520.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2520.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2539.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk new file mode 100644 index 000000000..65c02115b --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2539.tolk @@ -0,0 +1,9 @@ +import "@stdlib/tvm-dicts" +import "../imports/use-dicts-err" + +/** +@compilation_should_fail +@stderr tests/imports/use-dicts-err.tolk:2:22 +@stderr Using a non-imported symbol `createEmptyDict` +@stderr hint: forgot to import "@stdlib/tvm-dicts"? + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2540.tolk b/tolk-tester/tests/invalid-symbol/err-2540.tolk new file mode 100644 index 000000000..0b14df1b8 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2540.tolk @@ -0,0 +1,11 @@ +import "../imports/some-helpers" + +fun usingNonImported() { + var s: SomeStorage = {}; +} + +/** +@compilation_should_fail +@stderr Using a non-imported symbol `SomeStorage` +@stderr forgot to import "some-storage.tolk"? + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2619.tolk b/tolk-tester/tests/invalid-symbol/err-2619.tolk new file mode 100644 index 000000000..8eef3bc07 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2619.tolk @@ -0,0 +1,8 @@ +fun main() { + return random(); +} + +/** +@compilation_should_fail +@stderr `random` is not a function, you probably want `random.uint256()` + */ diff --git a/tolk-tester/tests/invalid-redefinition-3.tolk b/tolk-tester/tests/invalid-symbol/err-2649.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2649.tolk diff --git a/tolk-tester/tests/invalid-self-3.tolk b/tolk-tester/tests/invalid-symbol/err-2673.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2673.tolk diff --git a/tolk-tester/tests/invalid-redefinition-5.tolk b/tolk-tester/tests/invalid-symbol/err-2715.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-5.tolk rename to tolk-tester/tests/invalid-symbol/err-2715.tolk diff --git a/tolk-tester/tests/invalid-typing-1.tolk b/tolk-tester/tests/invalid-symbol/err-2750.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2750.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2770.tolk b/tolk-tester/tests/invalid-symbol/err-2770.tolk new file mode 100644 index 000000000..06935c93d --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2770.tolk @@ -0,0 +1,14 @@ + +fun int.demo(self) { + +} + +fun main() { + (10 as int?).demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found for type `int?` +@stderr (but it exists for type `int`) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2788.tolk b/tolk-tester/tests/invalid-symbol/err-2788.tolk new file mode 100644 index 000000000..64135e701 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2788.tolk @@ -0,0 +1,9 @@ +fun main() { + var f = stringHexToSlice; + f("asdf"); +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's compile-time only + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2804.tolk b/tolk-tester/tests/invalid-symbol/err-2804.tolk new file mode 100644 index 000000000..72f393eff --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2804.tolk @@ -0,0 +1,6 @@ +const asdf = ttt; + +/** +@compilation_should_fail +@stderr undefined symbol `ttt` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2833.tolk b/tolk-tester/tests/invalid-symbol/err-2833.tolk new file mode 100644 index 000000000..8a7d249ff --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2833.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + return Point.getDeclaredPackPrefix(); +} + +/** +@compilation_should_fail +@stderr type `Point` does not have a serialization prefix + */ diff --git a/tolk-tester/tests/invalid-redefinition-4.tolk b/tolk-tester/tests/invalid-symbol/err-2853.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-4.tolk rename to tolk-tester/tests/invalid-symbol/err-2853.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2898.tolk b/tolk-tester/tests/invalid-symbol/err-2898.tolk new file mode 100644 index 000000000..ba1de7751 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2898.tolk @@ -0,0 +1,16 @@ +fun main(a: int | slice) { + match (a) { + int => { + var cc: int = a; + }, + slice => { + return cc; + }, + }; +} + +/** +@compilation_should_fail +@stderr undefined symbol `cc` +@stderr return cc; + */ diff --git a/tolk-tester/tests/invalid-import.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk similarity index 73% rename from tolk-tester/tests/invalid-import.tolk rename to tolk-tester/tests/invalid-symbol/err-2980.tolk index 416764b62..de29cc657 100644 --- a/tolk-tester/tests/invalid-import.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -4,8 +4,8 @@ /** @compilation_should_fail -On Linux/Mac, `realpath()` returns an error, and the error message is `cannot find file` +On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr invalid-import.tolk:2:7: error: Failed to import: cannot +@stderr err-2980.tolk:2:7: error: Failed to import: cannot @stderr import "unexisting.tolk"; */ diff --git a/tolk-tester/tests/invalid-tolk-version.tolk b/tolk-tester/tests/invalid-syntax/err-3001.tolk similarity index 100% rename from tolk-tester/tests/invalid-tolk-version.tolk rename to tolk-tester/tests/invalid-syntax/err-3001.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3015.tolk b/tolk-tester/tests/invalid-syntax/err-3015.tolk new file mode 100644 index 000000000..11cce6035 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3015.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton(123); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3034.tolk b/tolk-tester/tests/invalid-syntax/err-3034.tolk new file mode 100644 index 000000000..af02af80a --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3034.tolk @@ -0,0 +1,8 @@ +struct A { + id; +} + +/** +@compilation_should_fail +@stderr expected `: `, got `;` +*/ diff --git a/tolk-tester/tests/invalid-bitwise-4.tolk b/tolk-tester/tests/invalid-syntax/err-3035.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3035.tolk diff --git a/tolk-tester/tests/invalid-nopar-1.tolk b/tolk-tester/tests/invalid-syntax/err-3041.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3041.tolk diff --git a/tolk-tester/tests/invalid-shift-1.tolk b/tolk-tester/tests/invalid-syntax/err-3044.tolk similarity index 100% rename from tolk-tester/tests/invalid-shift-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3044.tolk diff --git a/tolk-tester/tests/invalid-declaration-4.tolk b/tolk-tester/tests/invalid-syntax/err-3059.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3059.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3080.tolk b/tolk-tester/tests/invalid-syntax/err-3080.tolk new file mode 100644 index 000000000..e79e8ee95 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3080.tolk @@ -0,0 +1,11 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/invalid-bitwise-3.tolk b/tolk-tester/tests/invalid-syntax/err-3102.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3102.tolk diff --git a/tolk-tester/tests/invalid-catch-1.tolk b/tolk-tester/tests/invalid-syntax/err-3164.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3164.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3173.tolk b/tolk-tester/tests/invalid-syntax/err-3173.tolk new file mode 100644 index 000000000..2c3e23b7d --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3173.tolk @@ -0,0 +1,8 @@ +struct Pair`, got `{` + */ diff --git a/tolk-tester/tests/invalid-nopar-2.tolk b/tolk-tester/tests/invalid-syntax/err-3183.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3183.tolk diff --git a/tolk-tester/tests/invalid-nopar-4.tolk b/tolk-tester/tests/invalid-syntax/err-3197.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3197.tolk diff --git a/tolk-tester/tests/invalid-syntax-7.tolk b/tolk-tester/tests/invalid-syntax/err-3205.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3205.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3208.tolk b/tolk-tester/tests/invalid-syntax/err-3208.tolk new file mode 100644 index 000000000..75292b252 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3208.tolk @@ -0,0 +1,8 @@ +fun main() { + return address("0:gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg"); +} + +/** +@compilation_should_fail +@stderr invalid standard address + */ diff --git a/tolk-tester/tests/invalid-nopar-3.tolk b/tolk-tester/tests/invalid-syntax/err-3274.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3274.tolk diff --git a/tolk-tester/tests/invalid-const-1.tolk b/tolk-tester/tests/invalid-syntax/err-3295.tolk similarity index 100% rename from tolk-tester/tests/invalid-const-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3295.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3357.tolk b/tolk-tester/tests/invalid-syntax/err-3357.tolk new file mode 100644 index 000000000..de5569da3 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3357.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton("1000000000"); +} + +/** +@compilation_should_fail +@stderr argument is too big and leads to overflow + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3360.tolk b/tolk-tester/tests/invalid-syntax/err-3360.tolk new file mode 100644 index 000000000..7f1b46602 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3360.tolk @@ -0,0 +1,10 @@ + +fun main(r: int | slice) { + // after `is` we expect a type, parse `int?`, and expression becomes incorrect + return r is int ? 10 : 20; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `10` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3390.tolk b/tolk-tester/tests/invalid-syntax/err-3390.tolk new file mode 100644 index 000000000..404aa959c --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3390.tolk @@ -0,0 +1,8 @@ +fun main(): int { + ;; here not a comment +} + +/** +@compilation_should_fail +@stderr error: expected `;`, got `not` + */ diff --git a/tolk-tester/tests/invalid-cmt-nested.tolk b/tolk-tester/tests/invalid-syntax/err-3407.tolk similarity index 66% rename from tolk-tester/tests/invalid-cmt-nested.tolk rename to tolk-tester/tests/invalid-syntax/err-3407.tolk index 807e7be88..e37402deb 100644 --- a/tolk-tester/tests/invalid-cmt-nested.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3407.tolk @@ -7,5 +7,5 @@ not nested /** @compilation_should_fail -@stderr error: expected fun or get, got `*` +@stderr error: expected top-level declaration, got `*` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3419.tolk b/tolk-tester/tests/invalid-syntax/err-3419.tolk new file mode 100644 index 000000000..a59ca0e68 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3419.tolk @@ -0,0 +1,12 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + else => 0, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `else` branch should be the last + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3450.tolk b/tolk-tester/tests/invalid-syntax/err-3450.tolk new file mode 100644 index 000000000..5075284ec --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3450.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w3: Wrapper> = {}; +} + +/** +@compilation_should_fail +@stderr expected `>` or `,`, got `=` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk new file mode 100644 index 000000000..44c1b8954 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -0,0 +1,28 @@ +type asdf = int; +const gcon = 10; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + match (10) { + asdf => 2, // also match by type + }; + + match (10) { + gcon => 2, // match by expression (it's constant) + }; + + var ten = 10; + match (10) { + ten => 10, // not a constant, tries to lookup a type, fails + } +} + +/** +@compilation_should_fail +@stderr unknown type name `ten` +@stderr ten => 10 + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3472.tolk b/tolk-tester/tests/invalid-syntax/err-3472.tolk new file mode 100644 index 000000000..2d15d9fb1 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3472.tolk @@ -0,0 +1,6 @@ +const ss = ton("0.0.0"); + +/** +@compilation_should_fail +@stderr argument is not a valid number like "0.05" + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3482.tolk b/tolk-tester/tests/invalid-syntax/err-3482.tolk new file mode 100644 index 000000000..cfe2d3884 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3482.tolk @@ -0,0 +1,6 @@ +type MInt int; + +/** +@compilation_should_fail +@stderr expected `=`, got `int` +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3498.tolk b/tolk-tester/tests/invalid-syntax/err-3498.tolk new file mode 100644 index 000000000..204a750cc --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3498.tolk @@ -0,0 +1,12 @@ + +fun main() { + var z = 5; + ++z; + return z; +} + +/** +@compilation_should_fail +@stderr Tolk has no increment operator +@stderr hint: use `i += 1`, not `i++` + */ diff --git a/tolk-tester/tests/invalid-syntax-5.tolk b/tolk-tester/tests/invalid-syntax/err-3527.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3527.tolk diff --git a/tolk-tester/tests/invalid-syntax-1.tolk b/tolk-tester/tests/invalid-syntax/err-3551.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3551.tolk diff --git a/tolk-tester/tests/invalid-catch-2.tolk b/tolk-tester/tests/invalid-syntax/err-3586.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3586.tolk diff --git a/tolk-tester/tests/invalid-bitwise-5.tolk b/tolk-tester/tests/invalid-syntax/err-3588.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3588.tolk diff --git a/tolk-tester/tests/invalid-declaration-7.tolk b/tolk-tester/tests/invalid-syntax/err-3603.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3603.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3618.tolk b/tolk-tester/tests/invalid-syntax/err-3618.tolk new file mode 100644 index 000000000..0c3fa7d3f --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3618.tolk @@ -0,0 +1,6 @@ +const x = 0b0012; + +/** +@compilation_should_fail +@stderr expected top-level declaration, got `2` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3650.tolk b/tolk-tester/tests/invalid-syntax/err-3650.tolk new file mode 100644 index 000000000..5482e7b9c --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3650.tolk @@ -0,0 +1,8 @@ +struct A { + v1 = 0, +} + +/** +@compilation_should_fail +@stderr expected `: `, got `=` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3690.tolk b/tolk-tester/tests/invalid-syntax/err-3690.tolk new file mode 100644 index 000000000..934e18541 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3690.tolk @@ -0,0 +1,12 @@ +struct A { + id: int; +} + +fun main() { + A { id: 1, id: 2 }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3714.tolk b/tolk-tester/tests/invalid-syntax/err-3714.tolk new file mode 100644 index 000000000..a0f9bea13 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3714.tolk @@ -0,0 +1,8 @@ +fun main(am: slice) { + return ton(am); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-syntax-4.tolk b/tolk-tester/tests/invalid-syntax/err-3719.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3719.tolk diff --git a/tolk-tester/tests/invalid-declaration-5.tolk b/tolk-tester/tests/invalid-syntax/err-3781.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3781.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3810.tolk b/tolk-tester/tests/invalid-syntax/err-3810.tolk new file mode 100644 index 000000000..a70ae569b --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3810.tolk @@ -0,0 +1,14 @@ + +struct A { + f: int; +} + +fun main(a: A) { + a.f--; +} + +/** +@compilation_should_fail +@stderr Tolk has no decrement operator +@stderr hint: use `i -= 1`, not `i--` + */ diff --git a/tolk-tester/tests/invalid-bitwise-1.tolk b/tolk-tester/tests/invalid-syntax/err-3819.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3819.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3837.tolk b/tolk-tester/tests/invalid-syntax/err-3837.tolk new file mode 100644 index 000000000..c0ee50b30 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3837.tolk @@ -0,0 +1,14 @@ +fun f(x: int) { + match(x) { + -1 => -1, + 0 => 0, + else => 0, + else => -1 + }; +} + +/** +@compilation_should_fail +@stderr duplicated `else` branch +@stderr else => -1 + */ diff --git a/tolk-tester/tests/invalid-bitwise-7.tolk b/tolk-tester/tests/invalid-syntax/err-3851.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3851.tolk diff --git a/tolk-tester/tests/invalid-declaration-2.tolk b/tolk-tester/tests/invalid-syntax/err-3853.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3853.tolk diff --git a/tolk-tester/tests/invalid-syntax-2.tolk b/tolk-tester/tests/invalid-syntax/err-3898.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3898.tolk diff --git a/tolk-tester/tests/invalid-bitwise-2.tolk b/tolk-tester/tests/invalid-syntax/err-3910.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3910.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3913.tolk b/tolk-tester/tests/invalid-syntax/err-3913.tolk new file mode 100644 index 000000000..033743259 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3913.tolk @@ -0,0 +1,8 @@ +fun f(x: bool) { + return match (x) { true => 0, true => -1 }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/invalid-declaration-3.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk similarity index 51% rename from tolk-tester/tests/invalid-declaration-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3967.tolk index 3edc09fda..9a158a390 100644 --- a/tolk-tester/tests/invalid-declaration-3.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3967.tolk @@ -4,5 +4,5 @@ int main() { /** @compilation_should_fail -@stderr expected fun or get, got `int` +@stderr expected top-level declaration, got `int` */ diff --git a/tolk-tester/tests/invalid-bitwise-6.tolk b/tolk-tester/tests/invalid-syntax/err-3992.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-6.tolk rename to tolk-tester/tests/invalid-syntax/err-3992.tolk diff --git a/tolk-tester/tests/invalid-typing-10.tolk b/tolk-tester/tests/invalid-typing-10.tolk deleted file mode 100644 index 8c1df4a26..000000000 --- a/tolk-tester/tests/invalid-typing-10.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun failMathOnBoolean(c: cell) { - return (null == c) * 10; -} - -/** -@compilation_should_fail -@stderr can not apply operator `*` to `bool` and `int` - */ diff --git a/tolk-tester/tests/invalid-typing-12.tolk b/tolk-tester/tests/invalid-typing-12.tolk deleted file mode 100644 index 3a5b1fe28..000000000 --- a/tolk-tester/tests/invalid-typing-12.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun failAssignNullToTensor() { - var ab = (1, 2); - ab = null; - return ab; -} - -/** -@compilation_should_fail -@stderr can not assign `null` to variable of type `(int, int)` - */ diff --git a/tolk-tester/tests/invalid-typing-16.tolk b/tolk-tester/tests/invalid-typing-16.tolk deleted file mode 100644 index 1dca7822c..000000000 --- a/tolk-tester/tests/invalid-typing-16.tolk +++ /dev/null @@ -1,13 +0,0 @@ -@pure -fun myDictDeleteStrict(mutate self: cell, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; - - -fun testCantCallDictMethodsOnNullable(c: cell) { - c.beginParse().loadDict().myDictDeleteStrict(16, 1); -} - -/** -@compilation_should_fail -@stderr can not call method for `cell` with object of type `cell?` - */ diff --git a/tolk-tester/tests/invalid-typing-5.tolk b/tolk-tester/tests/invalid-typing-5.tolk deleted file mode 100644 index 9d8cd480d..000000000 --- a/tolk-tester/tests/invalid-typing-5.tolk +++ /dev/null @@ -1,13 +0,0 @@ -fun incNotChained(mutate self: int) { - self = self + 1; -} - -fun failWhenReturnANotChainedValue(x: int): int { - return x.incNotChained(); -} - -/** -@compilation_should_fail -@stderr x.incNotChained() -@stderr can not convert type `void` to return type `int` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6010.tolk b/tolk-tester/tests/invalid-typing/err-6010.tolk new file mode 100644 index 000000000..ece391173 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6010.tolk @@ -0,0 +1,12 @@ +fun analyze(t: slice) {} + +fun main(t: slice?) { + analyze(t); +} + +/** +@compilation_should_fail +@stderr can not pass `slice?` to `slice` +@stderr hint: probably, you should check on null +@stderr hint: alternatively, use `!` operator to bypass nullability checks: `!` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6013.tolk b/tolk-tester/tests/invalid-typing/err-6013.tolk new file mode 100644 index 000000000..c5ec6783d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6013.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.demoInvalid(self) { + return self.first == (null as int?); +} + +fun main(p: Pair) { + p.demoInvalid(); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.demoInvalid` + */ diff --git a/tolk-tester/tests/invalid-assign-1.tolk b/tolk-tester/tests/invalid-typing/err-6039.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-1.tolk rename to tolk-tester/tests/invalid-typing/err-6039.tolk diff --git a/tolk-tester/tests/invalid-assign-3.tolk b/tolk-tester/tests/invalid-typing/err-6040.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-3.tolk rename to tolk-tester/tests/invalid-typing/err-6040.tolk diff --git a/tolk-tester/tests/invalid-call-10.tolk b/tolk-tester/tests/invalid-typing/err-6041.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-10.tolk rename to tolk-tester/tests/invalid-typing/err-6041.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6070.tolk b/tolk-tester/tests/invalid-typing/err-6070.tolk new file mode 100644 index 000000000..e10560210 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6070.tolk @@ -0,0 +1,26 @@ +struct Wrapper { + value: T; +} + +struct JustInt { + value: int; +} + +fun takeSmth(obj: T) { return obj.value; } +fun takeW(obj: Wrapper) { return obj.value; } + +fun cantPassJustIntAsWrapper() { + var w: Wrapper = { value: 10 }; + var j: JustInt = { value: 10 }; + + takeSmth(w); + takeSmth(j); + takeW(w); + takeW(j); +} + +/** +@compilation_should_fail +@stderr can not pass `JustInt` to `Wrapper` +@stderr takeW(j) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6078.tolk b/tolk-tester/tests/invalid-typing/err-6078.tolk new file mode 100644 index 000000000..139086643 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6078.tolk @@ -0,0 +1,11 @@ +struct S { v: int; } + +fun cantAssignIncorrectTypeToField() { + var s: S = { v: 0 }; + s.v = createEmptyTuple(); +} + +/** +@compilation_should_fail +@stderr can not assign `tuple` to field of type `int` + */ diff --git a/tolk-tester/tests/invalid-typing-18.tolk b/tolk-tester/tests/invalid-typing/err-6080.tolk similarity index 88% rename from tolk-tester/tests/invalid-typing-18.tolk rename to tolk-tester/tests/invalid-typing/err-6080.tolk index cf985addf..d02675097 100644 --- a/tolk-tester/tests/invalid-typing-18.tolk +++ b/tolk-tester/tests/invalid-typing/err-6080.tolk @@ -1,5 +1,5 @@ fun incrementOrSetNull(mutate x: int?) { - if (random()) { x! += 1; } + if (random.uint256()) { x! += 1; } else { x = null; } } diff --git a/tolk-tester/tests/invalid-typing/err-6087.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk new file mode 100644 index 000000000..dfc3ee62f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6087.tolk @@ -0,0 +1,10 @@ +fun cantAutoCastBytesNToSlice() { + var b = beginCell().storeInt(1, 32).endCell().beginParse() as bits32; + return b.loadInt(32); +} + +/** +@compilation_should_fail +@stderr method `loadInt` not found for type `bits32` +@stderr but it exists for type `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6089.tolk b/tolk-tester/tests/invalid-typing/err-6089.tolk new file mode 100644 index 000000000..8548d1781 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6089.tolk @@ -0,0 +1,8 @@ +fun unionTypesNotAllowedInIs(a: int | slice | builder) { + if (a is int | slice) {} +} + +/** +@compilation_should_fail +@stderr union types are not allowed, use concrete types in `is` + */ diff --git a/tolk-tester/tests/invalid-generics-4.tolk b/tolk-tester/tests/invalid-typing/err-6095.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-4.tolk rename to tolk-tester/tests/invalid-typing/err-6095.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6110.tolk b/tolk-tester/tests/invalid-typing/err-6110.tolk new file mode 100644 index 000000000..49e9f30c7 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6110.tolk @@ -0,0 +1,15 @@ +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +fun cantDeduceDefiniteUnion(a: int | slice | builder) { + makeNullable(a); // ok + makeNullable(a); // ok + makeNullable(a); +} + +/** +@compilation_should_fail +@stderr can not deduce T1 for generic function `makeNullable` +@stderr makeNullable(a); + */ diff --git a/tolk-tester/tests/invalid-typing-20.tolk b/tolk-tester/tests/invalid-typing/err-6114.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-20.tolk rename to tolk-tester/tests/invalid-typing/err-6114.tolk diff --git a/tolk-tester/tests/invalid-typing-44.tolk b/tolk-tester/tests/invalid-typing/err-6130.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-44.tolk rename to tolk-tester/tests/invalid-typing/err-6130.tolk diff --git a/tolk-tester/tests/invalid-typing-4.tolk b/tolk-tester/tests/invalid-typing/err-6134.tolk similarity index 59% rename from tolk-tester/tests/invalid-typing-4.tolk rename to tolk-tester/tests/invalid-typing/err-6134.tolk index 1ee71290c..b3972a7af 100644 --- a/tolk-tester/tests/invalid-typing-4.tolk +++ b/tolk-tester/tests/invalid-typing/err-6134.tolk @@ -1,4 +1,4 @@ -fun incNotChained(mutate self: int) { +fun int.incNotChained(mutate self) { self = self + 1; } @@ -8,5 +8,5 @@ fun cantCallNotChainedMethodsInAChain(x: int) { /** @compilation_should_fail -@stderr can not call method for `int` with object of type `void` +@stderr method `incNotChained` not found for type `void` */ diff --git a/tolk-tester/tests/invalid-typing/err-6135.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk new file mode 100644 index 000000000..7bf580871 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6135.tolk @@ -0,0 +1,12 @@ +fun cantAssignIntNToCoins(n: int8, c: coins) { + n = c as int8; // ok + n = c as int; // ok + n = c; +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to variable of type `int8` +@stderr n = c; +@stderr hint: use `as` operator for unsafe casting: ` as int8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6144.tolk b/tolk-tester/tests/invalid-typing/err-6144.tolk new file mode 100644 index 000000000..6fe610eff --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6144.tolk @@ -0,0 +1,13 @@ +type IntOrSlice = int | slice; + +fun unionTypesNotAllowedInMatch(a: int | slice) { + match (a) { + IntOrSlice => 1, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: union types are not allowed, use concrete types in `match` +@stderr IntOrSlice + */ diff --git a/tolk-tester/tests/invalid-typing/err-6145.tolk b/tolk-tester/tests/invalid-typing/err-6145.tolk new file mode 100644 index 000000000..54893cb07 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6145.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + value: T; +} + +fun analyze(w: Wrapper) {} + +type MInt = int; + +fun main() { + var w1: Wrapper = { value: 10 }; + var w2 = Wrapper { value: 10 as int7 }; + + analyze(w1); + analyze(w2); +} + +/** +@compilation_should_fail +@stderr can not pass `Wrapper` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing-25.tolk b/tolk-tester/tests/invalid-typing/err-6148.tolk similarity index 83% rename from tolk-tester/tests/invalid-typing-25.tolk rename to tolk-tester/tests/invalid-typing/err-6148.tolk index 1621bab19..cbd305f21 100644 --- a/tolk-tester/tests/invalid-typing-25.tolk +++ b/tolk-tester/tests/invalid-typing/err-6148.tolk @@ -9,6 +9,6 @@ fun testSmartCastsDropAfterMutate() { /** @compilation_should_fail -@stderr type `(int, int)?` is not indexable +@stderr field `1` doesn't exist in type `(int, int)?` @stderr return x.1 */ diff --git a/tolk-tester/tests/invalid-typing/err-6149.tolk b/tolk-tester/tests/invalid-typing/err-6149.tolk new file mode 100644 index 000000000..7fd34b646 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6149.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +fun main(p: Pair) { + p.compareWith(null, null); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.compareWith` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6179.tolk b/tolk-tester/tests/invalid-typing/err-6179.tolk new file mode 100644 index 000000000..73148590b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6179.tolk @@ -0,0 +1,7 @@ +fun f(x: slice = ton("0.04")) { +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6184.tolk b/tolk-tester/tests/invalid-typing/err-6184.tolk new file mode 100644 index 000000000..16ba33862 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6184.tolk @@ -0,0 +1,11 @@ +struct S { v: () -> int; } + +fun cantCreateIncorrectTypeToField() { + var s: S = { v: beginCell }; +} + +/** +@compilation_should_fail +@stderr can not assign `() -> builder` to field of type `() -> int` +@stderr v: beginCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6188.tolk b/tolk-tester/tests/invalid-typing/err-6188.tolk new file mode 100644 index 000000000..7a7fb15e3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6188.tolk @@ -0,0 +1,11 @@ +type MInt = int; + +fun failMathOnBoolean(c: cell) { + var ten: MInt = 10; + return (null == c) * ten; +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `bool` and `MInt` + */ diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-typing/err-6200.tolk similarity index 59% rename from tolk-tester/tests/invalid-declaration-13.tolk rename to tolk-tester/tests/invalid-typing/err-6200.tolk index 758a4f21d..ada0ef5f4 100644 --- a/tolk-tester/tests/invalid-declaration-13.tolk +++ b/tolk-tester/tests/invalid-typing/err-6200.tolk @@ -2,6 +2,6 @@ const c: slice = 123 + 456; /** @compilation_should_fail -@stderr expression type does not match declared type +@stderr can not assign `int` to `slice` @stderr const c */ diff --git a/tolk-tester/tests/invalid-typing/err-6204.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk new file mode 100644 index 000000000..50c191f1f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6204.tolk @@ -0,0 +1,13 @@ +fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { + __expect_type(random.uint256() ? n : c as int8, "int8"); + __expect_type(random.uint256() ? n as int16 : c as int16, "int16"); + + var withHint: int = random.uint256() ? n : c; // ok + var withoutHint = random.uint256() ? n : c; // error +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int8` and `coins` +@stderr var withoutHint + */ diff --git a/tolk-tester/tests/invalid-typing-24.tolk b/tolk-tester/tests/invalid-typing/err-6220.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-24.tolk rename to tolk-tester/tests/invalid-typing/err-6220.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6230.tolk b/tolk-tester/tests/invalid-typing/err-6230.tolk new file mode 100644 index 000000000..03c80df87 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6230.tolk @@ -0,0 +1,10 @@ +struct S { a: int; b: int; } + +fun cantCastStructToTensor(s: S) { + s as (int, int); +} + +/** +@compilation_should_fail +@stderr type `S` can not be cast to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6249.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk new file mode 100644 index 000000000..f5f4722d9 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6249.tolk @@ -0,0 +1,11 @@ +fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { + __expect_type(random.uint256() ? n : c as slice, "slice"); + __expect_type(random.uint256() ? n : c as slice?, "slice?"); + random.uint256() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `slice` and `bytes16` +@stderr hint: maybe, you should use ` as ` to make them identical + */ diff --git a/tolk-tester/tests/invalid-self-5.tolk b/tolk-tester/tests/invalid-typing/err-6257.tolk similarity index 66% rename from tolk-tester/tests/invalid-self-5.tolk rename to tolk-tester/tests/invalid-typing/err-6257.tolk index a007a93c2..5e18b376c 100644 --- a/tolk-tester/tests/invalid-self-5.tolk +++ b/tolk-tester/tests/invalid-typing/err-6257.tolk @@ -1,9 +1,9 @@ -fun increment(mutate self: int): self { +fun int.increment(mutate self): self { self = self + 1; return self; } -fun cantReturnAnotherSelf(mutate self: int): self { +fun int.cantReturnAnotherSelf(mutate self): self { self = self + 1; var x = 0; return x.increment(); diff --git a/tolk-tester/tests/invalid-typing/err-6274.tolk b/tolk-tester/tests/invalid-typing/err-6274.tolk new file mode 100644 index 000000000..d586efe41 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6274.tolk @@ -0,0 +1,19 @@ +type MInt = int; +type MInt2 = MInt | int; +type MSlice = slice; +type MCell = cell; +type MBuilder = builder; + +fun cantMatchAgainstNotVariant(a: int | slice | MBuilder) { + match (a) { + MInt2 => 10, + MSlice => 20, + MCell => 30, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `cell` is not a variant of `int | slice | MBuilder` +@stderr MCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6281.tolk b/tolk-tester/tests/invalid-typing/err-6281.tolk new file mode 100644 index 000000000..7d77f411f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6281.tolk @@ -0,0 +1,12 @@ +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } + +fun main() { + Wrapper>.create(123); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-self-1.tolk b/tolk-tester/tests/invalid-typing/err-6288.tolk similarity index 65% rename from tolk-tester/tests/invalid-self-1.tolk rename to tolk-tester/tests/invalid-typing/err-6288.tolk index 40b54f163..3ea5c02a1 100644 --- a/tolk-tester/tests/invalid-self-1.tolk +++ b/tolk-tester/tests/invalid-typing/err-6288.tolk @@ -1,4 +1,4 @@ -fun cantReturnFromSelf(mutate self: int): self { +fun int.cantReturnFromSelf(mutate self): self { return 2; } diff --git a/tolk-tester/tests/invalid-typing/err-6353.tolk b/tolk-tester/tests/invalid-typing/err-6353.tolk new file mode 100644 index 000000000..fb2537592 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6353.tolk @@ -0,0 +1,30 @@ +fun cantInferUnionTypeWithoutHint(a: int | slice | builder) { + var ok1: builder | (slice | int | int) = match (a) { + int => a, + slice => a, + builder => a, + }; + var ok2 = match (a) { + int => a, + slice => a, + builder => a, + } as builder | (slice | int | int); + var ok3 = match (a) { + int => a as builder | slice | int, + slice => a as builder | slice | int, + builder => a as builder | slice, + }; + + var err = match (a) { + int => a!, + slice => a!, + builder => a!, + }; +} + +/** +@compilation_should_fail +@stderr type of `match` was inferred as `int | slice | builder`; probably, it's not what you expected +@stderr assign it to a variable `var v: = match (...) { ... }` manually +@stderr err = match (a) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6366.tolk b/tolk-tester/tests/invalid-typing/err-6366.tolk new file mode 100644 index 000000000..cba32e770 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6366.tolk @@ -0,0 +1,14 @@ + +fun main() { + match (0 as int | slice) { + int => 1, + slice => 2, + else => 10, + }; +} + +/** +@compilation_should_fail +@stderr `else` is not allowed in `match` by type; you should cover all possible types +@stderr else => 10 + */ diff --git a/tolk-tester/tests/invalid-typing-45.tolk b/tolk-tester/tests/invalid-typing/err-6368.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-45.tolk rename to tolk-tester/tests/invalid-typing/err-6368.tolk diff --git a/tolk-tester/tests/invalid-generics-3.tolk b/tolk-tester/tests/invalid-typing/err-6369.tolk similarity index 68% rename from tolk-tester/tests/invalid-generics-3.tolk rename to tolk-tester/tests/invalid-typing/err-6369.tolk index 72b7df0ec..3d9391b11 100644 --- a/tolk-tester/tests/invalid-generics-3.tolk +++ b/tolk-tester/tests/invalid-typing/err-6369.tolk @@ -6,6 +6,6 @@ fun failIncompatibleTypesForT() { /** @compilation_should_fail -@stderr T is both int and slice for generic function `f` +@stderr can not pass `slice` to `int` @stderr f(32 */ diff --git a/tolk-tester/tests/invalid-typing-23.tolk b/tolk-tester/tests/invalid-typing/err-6370.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-23.tolk rename to tolk-tester/tests/invalid-typing/err-6370.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6380.tolk b/tolk-tester/tests/invalid-typing/err-6380.tolk new file mode 100644 index 000000000..b7ab81a33 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6380.tolk @@ -0,0 +1,14 @@ +struct A { + f1: never; +} + +fun main() { + var a: A = {}; // f1 is never, can be omitted + a.f1 = 100; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `never` +@stderr a.f1 = 100 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6390.tolk b/tolk-tester/tests/invalid-typing/err-6390.tolk new file mode 100644 index 000000000..969cfd843 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6390.tolk @@ -0,0 +1,8 @@ +fun main(k: (int, slice)) { + var (a, b, c: int?) = k; +} + +/** +@compilation_should_fail +@stderr can not assign `(int, slice)`, sizes mismatch + */ diff --git a/tolk-tester/tests/invalid-mutate-20.tolk b/tolk-tester/tests/invalid-typing/err-6391.tolk similarity index 82% rename from tolk-tester/tests/invalid-mutate-20.tolk rename to tolk-tester/tests/invalid-typing/err-6391.tolk index f6eb2f9fd..1da57d883 100644 --- a/tolk-tester/tests/invalid-mutate-20.tolk +++ b/tolk-tester/tests/invalid-typing/err-6391.tolk @@ -1,4 +1,4 @@ -fun acceptMutateNullableTensor(mutate self: (int, int)?) { +fun (int, int)?.acceptMutateNullableTensor(mutate self) { } fun cantModifyTupleIndexWithTypeTransition() { diff --git a/tolk-tester/tests/invalid-typing-9.tolk b/tolk-tester/tests/invalid-typing/err-6393.tolk similarity index 85% rename from tolk-tester/tests/invalid-typing-9.tolk rename to tolk-tester/tests/invalid-typing/err-6393.tolk index a0d5ee04e..28a9f76ca 100644 --- a/tolk-tester/tests/invalid-typing-9.tolk +++ b/tolk-tester/tests/invalid-typing/err-6393.tolk @@ -1,3 +1,6 @@ +fun tupleLast(t: tuple): T + asm "LAST"; + fun getTupleLastGetter(): tuple -> X { return tupleLast; } diff --git a/tolk-tester/tests/invalid-typing/err-6403.tolk b/tolk-tester/tests/invalid-typing/err-6403.tolk new file mode 100644 index 000000000..ed40fc48d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6403.tolk @@ -0,0 +1,16 @@ +fun assign0(mutate x: uint8) { + x = 0; +} + +fun testCantPassDifferentIntN(x: int8) { + assign0(mutate (x as int)); // ok + assign0(mutate (x as uint8)); // ok + assign0(mutate ((x as int64) as int)); // ok + assign0(mutate x); +} + +/** +@compilation_should_fail +@stderr can not pass `int8` to `uint8` +@stderr assign0(mutate x); + */ diff --git a/tolk-tester/tests/invalid-never-1.tolk b/tolk-tester/tests/invalid-typing/err-6407.tolk similarity index 76% rename from tolk-tester/tests/invalid-never-1.tolk rename to tolk-tester/tests/invalid-typing/err-6407.tolk index 68c6c8043..8f0265de2 100644 --- a/tolk-tester/tests/invalid-never-1.tolk +++ b/tolk-tester/tests/invalid-typing/err-6407.tolk @@ -1,5 +1,5 @@ fun invalidNever(): never { - if (random()) { throw 123; } + if (random.uint256()) { throw 123; } } /** diff --git a/tolk-tester/tests/invalid-typing/err-6409.tolk b/tolk-tester/tests/invalid-typing/err-6409.tolk new file mode 100644 index 000000000..93c347edf --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6409.tolk @@ -0,0 +1,10 @@ +struct Wrapper { value: T } + +fun f(w: Wrapper | Wrapper) { + w = Wrapper { value: beginCell() }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6427.tolk b/tolk-tester/tests/invalid-typing/err-6427.tolk new file mode 100644 index 000000000..37df89a1e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6427.tolk @@ -0,0 +1,13 @@ +fun cantReturnVoidAndNonVoid(a: int | slice | builder) { + match (a) { + int => { return; } + slice => {} + builder => "" + }; + return 123; +} + +/** +@compilation_should_fail +@stderr mixing void and non-void returns in function + */ diff --git a/tolk-tester/tests/invalid-typing-2.tolk b/tolk-tester/tests/invalid-typing/err-6450.tolk similarity index 65% rename from tolk-tester/tests/invalid-typing-2.tolk rename to tolk-tester/tests/invalid-typing/err-6450.tolk index 052596e4c..b14f3b6d5 100644 --- a/tolk-tester/tests/invalid-typing-2.tolk +++ b/tolk-tester/tests/invalid-typing/err-6450.tolk @@ -1,9 +1,11 @@ +type MInt = int; + fun main() { - var tri: (int, int) = (10, false); + var tri: (int, MInt) = (10, false); return; } /** @compilation_should_fail -@stderr can not assign `(int, bool)` to variable of type `(int, int)` +@stderr can not assign `(int, bool)` to variable of type `(int, MInt)` */ diff --git a/tolk-tester/tests/invalid-typing/err-6460.tolk b/tolk-tester/tests/invalid-typing/err-6460.tolk new file mode 100644 index 000000000..554f8b4d3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6460.tolk @@ -0,0 +1,19 @@ +type MNull = null; +type MNull2 = MNull; + +fun main(t: MNull2) { + t as null; + t as MNull; + t as MNull2; + t as int?; + t as int | slice | null; + t as (int, int) | slice | MNull; + + t as int; // error +} + +/** +@compilation_should_fail +@stderr type `null` can not be cast to `int` +@stderr t as int + */ diff --git a/tolk-tester/tests/invalid-typing/err-6473.tolk b/tolk-tester/tests/invalid-typing/err-6473.tolk new file mode 100644 index 000000000..a6859e03b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6473.tolk @@ -0,0 +1,18 @@ +struct Wrapper { + value: T; +} + +type IntOrWrapped8 = Wrapper | int; + +fun main() { + var w1: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int8 }; + var w2: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int16 }; + + var w3: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int32 }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | int | Wrapper` +@stderr var w3 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6480.tolk b/tolk-tester/tests/invalid-typing/err-6480.tolk new file mode 100644 index 000000000..25a85e0bc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6480.tolk @@ -0,0 +1,27 @@ +struct All { + f1: A; + f2: A; + f3: A; +} + +fun testInferGenericOneByOne(x: int?) { + // ok + __expect_type(All { + f1: x, + f2: 5, + f3: null + }, "All"); + // err + All { + f1: x, + f2: null, + f3: beginCell() // type checker error + }; +} + +/** +@compilation_should_fail +@stderr can not assign `builder` to field of type `int?` +@stderr f3: beginCell() + */ + diff --git a/tolk-tester/tests/invalid-typing-29.tolk b/tolk-tester/tests/invalid-typing/err-6496.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-29.tolk rename to tolk-tester/tests/invalid-typing/err-6496.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6498.tolk b/tolk-tester/tests/invalid-typing/err-6498.tolk new file mode 100644 index 000000000..79da03b65 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6498.tolk @@ -0,0 +1,14 @@ + +fun main() { + var a = 0 as int | slice; + var bb = match (a) { + }; + __expect_type(a, "int | slice"); + return a; +} + +/** +@compilation_should_fail +@stderr empty `match` can't be used as expression +@stderr var bb = match + */ diff --git a/tolk-tester/tests/invalid-typing-22.tolk b/tolk-tester/tests/invalid-typing/err-6499.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-22.tolk rename to tolk-tester/tests/invalid-typing/err-6499.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6502.tolk b/tolk-tester/tests/invalid-typing/err-6502.tolk new file mode 100644 index 000000000..2f9e664d8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6502.tolk @@ -0,0 +1,11 @@ +fun cantAutoInferUnionVariant() { + var a: (int?, int?) | (int, int?) | int = 5; + a = (1, 2) as (int?, int?); // ok + a = (1 as int?, 2 as int?); // ok + a = (1, 2); +} + +/** +@compilation_should_fail +@stderr can not assign `(int, int)` to variable of type `(int?, int?) | (int, int?) | int` + */ diff --git a/tolk-tester/tests/invalid-generics-13.tolk b/tolk-tester/tests/invalid-typing/err-6509.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-13.tolk rename to tolk-tester/tests/invalid-typing/err-6509.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6518.tolk b/tolk-tester/tests/invalid-typing/err-6518.tolk new file mode 100644 index 000000000..382767744 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6518.tolk @@ -0,0 +1,16 @@ +type MNull = null; +type IntOr16 = int | int16; + +fun main(t: int?) { + t as int? | null; + t as int? | slice; + t as (int, int) | MNull | int8 | IntOr16; + + t as slice?; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `slice?` +@stderr t as slice? + */ diff --git a/tolk-tester/tests/invalid-self-2.tolk b/tolk-tester/tests/invalid-typing/err-6533.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-2.tolk rename to tolk-tester/tests/invalid-typing/err-6533.tolk diff --git a/tolk-tester/tests/invalid-typing-26.tolk b/tolk-tester/tests/invalid-typing/err-6553.tolk similarity index 61% rename from tolk-tester/tests/invalid-typing-26.tolk rename to tolk-tester/tests/invalid-typing/err-6553.tolk index bf5a1165c..ba6807713 100644 --- a/tolk-tester/tests/invalid-typing-26.tolk +++ b/tolk-tester/tests/invalid-typing/err-6553.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MInt = int; + +fun getNullableInt(): MInt? { return 5; } fun testAssertThrowIsConditional() { var (x, y) = (getNullableInt(), getNullableInt()); @@ -8,5 +10,5 @@ fun testAssertThrowIsConditional() { /** @compilation_should_fail -@stderr can not apply operator `+` to `int` and `int?` +@stderr can not apply operator `+` to `MInt` and `MInt?` */ diff --git a/tolk-tester/tests/invalid-typing-21.tolk b/tolk-tester/tests/invalid-typing/err-6580.tolk similarity index 78% rename from tolk-tester/tests/invalid-typing-21.tolk rename to tolk-tester/tests/invalid-typing/err-6580.tolk index d2a815eee..e2c088b04 100644 --- a/tolk-tester/tests/invalid-typing-21.tolk +++ b/tolk-tester/tests/invalid-typing/err-6580.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MIntN = int?; + +fun getNullableInt(): MIntN { return 5; } fun testNeverTypeOccurs() { var x: int? = getNullableInt(); diff --git a/tolk-tester/tests/invalid-typing/err-6583.tolk b/tolk-tester/tests/invalid-typing/err-6583.tolk new file mode 100644 index 000000000..530ff95cc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6583.tolk @@ -0,0 +1,12 @@ +fun cantAssignIncompatibleUnionTypes(a: int | slice | builder) { + var b: int | slice = match (a) { + int => a, + slice => a, + builder => null, + }; +} + +/** +@compilation_should_fail +@stderr can not assign `int | slice | null` to variable of type `int | slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6595.tolk b/tolk-tester/tests/invalid-typing/err-6595.tolk new file mode 100644 index 000000000..c016ae187 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6595.tolk @@ -0,0 +1,10 @@ +fun main() { + blockchain.configParam(true); +} + +/** +@compilation_should_fail +@stderr can not pass `bool` to `int` +@stderr hint: use `as` operator for unsafe casting: ` as int` +@stderr caution! in TVM, bool TRUE is -1, not 1 + */ diff --git a/tolk-tester/tests/invalid-typing-8.tolk b/tolk-tester/tests/invalid-typing/err-6600.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-8.tolk rename to tolk-tester/tests/invalid-typing/err-6600.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6603.tolk b/tolk-tester/tests/invalid-typing/err-6603.tolk new file mode 100644 index 000000000..5f6275c35 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6603.tolk @@ -0,0 +1,14 @@ +struct A { + kk: tuple + dd: [int, int] +} + +fun main(a: A) { + a.kk = a.dd +} + +/** +@compilation_should_fail +@stderr can not assign `[int, int]` to field of type `tuple` +@stderr hint: use `as` operator for unsafe casting: ` as tuple` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6613.tolk b/tolk-tester/tests/invalid-typing/err-6613.tolk new file mode 100644 index 000000000..240d5b398 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6613.tolk @@ -0,0 +1,10 @@ +fun testCantCastBoolToUIntN(x: int) { + var eq = x == 0; + eq as int8; // ok + eq as uint8; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `uint8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6618.tolk b/tolk-tester/tests/invalid-typing/err-6618.tolk new file mode 100644 index 000000000..ab505c124 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6618.tolk @@ -0,0 +1,17 @@ +struct Ok { result: T } +struct Err { errPayload: T } + +type Response = Ok | Err; + +fun f(w: Response | Response) { + match (w) { + Ok => {} + Err => {} + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types +@stderr missing types are: `Ok`, `Err` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6619.tolk b/tolk-tester/tests/invalid-typing/err-6619.tolk new file mode 100644 index 000000000..b43aaef55 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6619.tolk @@ -0,0 +1,6 @@ +const asdf = 1 + ""; + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6620.tolk b/tolk-tester/tests/invalid-typing/err-6620.tolk new file mode 100644 index 000000000..47e30e99c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6620.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +fun main() { + var c1: Wrapper< Wrapper<(int,int)> > = {value:{value:(1,2)}}; // ok + var c2: Wrapper< Wrapper<(int?,int?)> > = {value:{value:(null,null)}}; // ok + + var c3: Wrapper< Wrapper<(int,int)> > = {value:{value:0}}; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `(int, int)` +@stderr {value:0} + */ diff --git a/tolk-tester/tests/invalid-generics-8.tolk b/tolk-tester/tests/invalid-typing/err-6629.tolk similarity index 73% rename from tolk-tester/tests/invalid-generics-8.tolk rename to tolk-tester/tests/invalid-typing/err-6629.tolk index d2c24e532..c68082fa6 100644 --- a/tolk-tester/tests/invalid-generics-8.tolk +++ b/tolk-tester/tests/invalid-typing/err-6629.tolk @@ -6,6 +6,6 @@ fun wrongTCountPassed() { /** @compilation_should_fail -@stderr wrong count of generic T: expected 2, got 1 +@stderr expected 2 type arguments, got 1 @stderr */ diff --git a/tolk-tester/tests/invalid-typing/err-6633.tolk b/tolk-tester/tests/invalid-typing/err-6633.tolk new file mode 100644 index 000000000..d8ed9c298 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6633.tolk @@ -0,0 +1,11 @@ +fun main(c: cell) { + var sender_address: slice = c.beginParse().loadAddress(); +} + +/** +@compilation_should_fail +@stderr can not assign `address` to variable of type `slice` +@stderr hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level); +@stderr most likely, you just need `address` everywhere +@stderr hint: alternatively, use `as` operator for unsafe casting: ` as slice` + */ diff --git a/tolk-tester/tests/invalid-typing-14.tolk b/tolk-tester/tests/invalid-typing/err-6637.tolk similarity index 84% rename from tolk-tester/tests/invalid-typing-14.tolk rename to tolk-tester/tests/invalid-typing/err-6637.tolk index 657ab5f4a..26c7ce108 100644 --- a/tolk-tester/tests/invalid-typing-14.tolk +++ b/tolk-tester/tests/invalid-typing/err-6637.tolk @@ -1,6 +1,6 @@ fun autoGetIntOrNull() { - if (random()) { return 1; } + if (random.uint256()) { return 1; } return null; } diff --git a/tolk-tester/tests/invalid-typing/err-6642.tolk b/tolk-tester/tests/invalid-typing/err-6642.tolk new file mode 100644 index 000000000..82563e322 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6642.tolk @@ -0,0 +1,21 @@ +struct Wrapper { + value: T; +} + +type MyInt = int; + +fun cantAssignFieldsOfDifferentTypes( + w1: Wrapper, w2: Wrapper, w3: Wrapper, w4: Wrapper<()> +) { + w1.value = w2.value; // ok + w1.value = w3.value; // ok + w3.value = w2.value; // ok + + w1.value = w4.value; +} + +/** +@compilation_should_fail +@stderr can not assign `()` to field of type `int` +@stderr w1.value = w4.value + */ diff --git a/tolk-tester/tests/invalid-typing/err-6659.tolk b/tolk-tester/tests/invalid-typing/err-6659.tolk new file mode 100644 index 000000000..326e239d8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6659.tolk @@ -0,0 +1,20 @@ +fun rand() { return random.uint256() } + +fun cantAutoInferUnionInTernary() { + var dd: int | slice = rand() ? 1 : ""; // ok + var cc = (rand() ? 1 : "") as int|slice; // ok + rand() ? 1 as int|slice : "" as int|slice; // ok + rand() ? 1 as int|slice : ""; // ok + rand() ? 1 : "" as int|slice|builder; // ok + rand() ? 1 as int|slice|null : "" as slice?; // ok + + var sub = rand() ? (1, 2 as int|slice) : (1, ""); + __expect_type(sub, "(int, int | slice)"); + + rand() ? 1 : ""; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6673.tolk b/tolk-tester/tests/invalid-typing/err-6673.tolk new file mode 100644 index 000000000..59a6ba283 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6673.tolk @@ -0,0 +1,14 @@ +fun cantAutoInferUnionVariant() { + var a: int8 | int16 = 0 as int8; // ok + a = 0 as int16; // ok + if (a is int8) { + a = 1 as int8; // ok + } + a = 2; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to variable of type `int8 | int16` +@stderr a = 2; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6690.tolk b/tolk-tester/tests/invalid-typing/err-6690.tolk new file mode 100644 index 000000000..cfbcd8cf4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6690.tolk @@ -0,0 +1,12 @@ +fun main(t: int?) { + t!; + t! as int; + + t as int; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `int` +@stderr t as int; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6709.tolk b/tolk-tester/tests/invalid-typing/err-6709.tolk new file mode 100644 index 000000000..d1fe41281 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6709.tolk @@ -0,0 +1,13 @@ +struct A { id: int; } +struct B { id: int; } +type AAlias = A; + +fun cantAssignDifferentStructures(a: A) { + var a2: AAlias = a; // ok + var b: B = a; +} + +/** +@compilation_should_fail +@stderr can not assign `A` to variable of type `B` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6710.tolk b/tolk-tester/tests/invalid-typing/err-6710.tolk new file mode 100644 index 000000000..d0fae1d56 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6710.tolk @@ -0,0 +1,15 @@ +fun f(s: slice) { + +} + +fun testCantPassAddressToSlice() { + var a = createAddressNone(); + f(a as slice); // ok + f(a); +} + +/** +@compilation_should_fail +@stderr can not pass `address` to `slice` +@stderr f(a); + */ diff --git a/tolk-tester/tests/invalid-typing-11.tolk b/tolk-tester/tests/invalid-typing/err-6711.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-11.tolk rename to tolk-tester/tests/invalid-typing/err-6711.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6712.tolk b/tolk-tester/tests/invalid-typing/err-6712.tolk new file mode 100644 index 000000000..c0210a7bc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6712.tolk @@ -0,0 +1,12 @@ +fun matchDoesntCoverAllCases(a: int | slice?) { + match (a) { + int => {} + slice => {} + }; +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `null` +@stderr match (a) + */ diff --git a/tolk-tester/tests/invalid-typing-27.tolk b/tolk-tester/tests/invalid-typing/err-6713.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-27.tolk rename to tolk-tester/tests/invalid-typing/err-6713.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6716.tolk b/tolk-tester/tests/invalid-typing/err-6716.tolk new file mode 100644 index 000000000..134fb9d14 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6716.tolk @@ -0,0 +1,7 @@ +fun f(d: bits8 = "") {} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `bits8` +@stderr hint: use `as` operator for unsafe casting: ` as bits8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6722.tolk b/tolk-tester/tests/invalid-typing/err-6722.tolk new file mode 100644 index 000000000..854e21167 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6722.tolk @@ -0,0 +1,7 @@ +import "../imports/./has-tinf-err" + +/** +@compilation_should_fail +@stderr imports/has-tinf-err.tolk:3:18 +@stderr can not assign `slice` to variable of type `int` + */ diff --git a/tolk-tester/tests/invalid-assign-2.tolk b/tolk-tester/tests/invalid-typing/err-6731.tolk similarity index 54% rename from tolk-tester/tests/invalid-assign-2.tolk rename to tolk-tester/tests/invalid-typing/err-6731.tolk index 6a33e6968..461e48d55 100644 --- a/tolk-tester/tests/invalid-assign-2.tolk +++ b/tolk-tester/tests/invalid-typing/err-6731.tolk @@ -4,5 +4,5 @@ fun main(cs: slice) { /** @compilation_should_fail -@stderr referencing a method for `tuple` with object of type `slice` +@stderr field `tupleSize` doesn't exist in type `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6734.tolk b/tolk-tester/tests/invalid-typing/err-6734.tolk new file mode 100644 index 000000000..0f805cbe5 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6734.tolk @@ -0,0 +1,12 @@ +type MTensor = (int, int); + +fun failAssignNullToTensor() { + var ab: MTensor = (1, 2); + ab = null; + return ab; +} + +/** +@compilation_should_fail +@stderr can not assign `null` to variable of type `MTensor` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk new file mode 100644 index 000000000..86934f3c8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -0,0 +1,13 @@ +type MInt32 = int32; + +fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { + op = qid as MInt32; // ok + op = qid; +} + +/** +@compilation_should_fail +@stderr can not assign `uint64` to variable of type `int32` +@stderr op = qid; +@stderr hint: use `as` operator for unsafe casting: ` as int32` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6740.tolk b/tolk-tester/tests/invalid-typing/err-6740.tolk new file mode 100644 index 000000000..e3dd3a470 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6740.tolk @@ -0,0 +1,9 @@ +struct A { + v1: int = "asdf"; +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `int` +@stderr v1: int = "asdf" + */ diff --git a/tolk-tester/tests/invalid-typing-6.tolk b/tolk-tester/tests/invalid-typing/err-6743.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-6.tolk rename to tolk-tester/tests/invalid-typing/err-6743.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6767.tolk b/tolk-tester/tests/invalid-typing/err-6767.tolk new file mode 100644 index 000000000..f9ddf7273 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6767.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + return (5 as int8?) as MInt | slice | null; +} + +/** +@compilation_should_fail +@stderr type `int8?` can not be cast to `MInt | slice | null` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6801.tolk b/tolk-tester/tests/invalid-typing/err-6801.tolk new file mode 100644 index 000000000..4646bd7be --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6801.tolk @@ -0,0 +1,9 @@ + +fun main(a: slice | null | int8) { + a as slice | null; +} + +/** +@compilation_should_fail +@stderr type `slice | null | int8` can not be cast to `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6803.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk new file mode 100644 index 000000000..1e9592a68 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6803.tolk @@ -0,0 +1,13 @@ +fun takeAnySlice(s: slice) {} + +fun cantPassBytesNToSlice(c: bits16) { + takeAnySlice(c as slice); // ok + takeAnySlice(c); +} + +/** +@compilation_should_fail +@stderr can not pass `bits16` to `slice` +@stderr takeAnySlice(c); +@stderr hint: use `as` operator for unsafe casting: ` as slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6808.tolk b/tolk-tester/tests/invalid-typing/err-6808.tolk new file mode 100644 index 000000000..46db0d6de --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6808.tolk @@ -0,0 +1,11 @@ +fun main() { + (1 as int16) as (int | int16); // ok, exact match + (1 as int8) as (int | int32); // ok, int accepts int8, but int32 not + + 1 as (int16 | int32); // error, ambiguous +} + +/** +@compilation_should_fail +@stderr type `int` can not be cast to `int16 | int32` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6810.tolk b/tolk-tester/tests/invalid-typing/err-6810.tolk new file mode 100644 index 000000000..d21099eb8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6810.tolk @@ -0,0 +1,12 @@ +fun cantCastBoolToCoins() { + var k1 = true as int8; // ok + var k2 = true as int1; // ok + var k3 = (true as int) as coins; // ok + var k4 = true as coins; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `coins` +@stderr true as coins + */ diff --git a/tolk-tester/tests/invalid-typing/err-6820.tolk b/tolk-tester/tests/invalid-typing/err-6820.tolk new file mode 100644 index 000000000..092858290 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6820.tolk @@ -0,0 +1,9 @@ +fun cantUnifyBytesNAndBytesM(n: bytes8, c: bytes16) { + __expect_type(random.uint256() ? n : c as bytes8, "bytes8"); + random.uint256() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `bytes8` and `bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6829.tolk b/tolk-tester/tests/invalid-typing/err-6829.tolk new file mode 100644 index 000000000..d3f2e505c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6829.tolk @@ -0,0 +1,18 @@ +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); + +struct A {} + +fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | A) { + if (a !is slice) { + match (a) { + (int, int) => {} + [[int]] => {} + } + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `A` + */ diff --git a/tolk-tester/tests/invalid-typing-3.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk similarity index 53% rename from tolk-tester/tests/invalid-typing-3.tolk rename to tolk-tester/tests/invalid-typing/err-6839.tolk index ac019a421..e8a4fd7c0 100644 --- a/tolk-tester/tests/invalid-typing-3.tolk +++ b/tolk-tester/tests/invalid-typing/err-6839.tolk @@ -1,9 +1,9 @@ -fun incInt(mutate self: int): self { +fun int.incInt(mutate self): self { self += 1; return self; } -fun appendBuilder(mutate self: builder): self { +fun builder.appendBuilder(mutate self): self { self.storeUint(1, 32); return self; } @@ -15,5 +15,6 @@ fun cantMixDifferentThis() { /** @compilation_should_fail -@stderr can not call method for `builder` with object of type `int` +@stderr method `appendBuilder` not found for type `int` +@stderr (but it exists for type `builder`) */ diff --git a/tolk-tester/tests/invalid-typing-15.tolk b/tolk-tester/tests/invalid-typing/err-6840.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-15.tolk rename to tolk-tester/tests/invalid-typing/err-6840.tolk diff --git a/tolk-tester/tests/invalid-typing-28.tolk b/tolk-tester/tests/invalid-typing/err-6841.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-28.tolk rename to tolk-tester/tests/invalid-typing/err-6841.tolk diff --git a/tolk-tester/tests/invalid-self-4.tolk b/tolk-tester/tests/invalid-typing/err-6844.tolk similarity index 61% rename from tolk-tester/tests/invalid-self-4.tolk rename to tolk-tester/tests/invalid-typing/err-6844.tolk index 0be6b9e4d..3f719f204 100644 --- a/tolk-tester/tests/invalid-self-4.tolk +++ b/tolk-tester/tests/invalid-typing/err-6844.tolk @@ -1,4 +1,4 @@ -fun cantReturnNothingFromSelf(mutate self: int): self { +fun int.cantReturnNothingFromSelf(mutate self): self { self = self + 1; } diff --git a/tolk-tester/tests/invalid-typing/err-6845.tolk b/tolk-tester/tests/invalid-typing/err-6845.tolk new file mode 100644 index 000000000..b739ce52a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6845.tolk @@ -0,0 +1,12 @@ +struct A { id: int; value: int; } + +fun takeTensor(v: (int, int)) {} + +fun cantPassStructAsTensor() { + takeTensor(A{id: 1, value:2}); +} + +/** +@compilation_should_fail +@stderr can not pass `A` to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6853.tolk b/tolk-tester/tests/invalid-typing/err-6853.tolk new file mode 100644 index 000000000..f901ad48f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6853.tolk @@ -0,0 +1,8 @@ +fun cantAssignIncompatiblePrimitiveNullable(m: int?) { + var a: slice? = m; +} + +/** +@compilation_should_fail +@stderr can not assign `int?` to variable of type `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing-17.tolk b/tolk-tester/tests/invalid-typing/err-6881.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-17.tolk rename to tolk-tester/tests/invalid-typing/err-6881.tolk diff --git a/tolk-tester/tests/invalid-typing-7.tolk b/tolk-tester/tests/invalid-typing/err-6901.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-7.tolk rename to tolk-tester/tests/invalid-typing/err-6901.tolk diff --git a/tolk-tester/tests/invalid-typing-19.tolk b/tolk-tester/tests/invalid-typing/err-6903.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-19.tolk rename to tolk-tester/tests/invalid-typing/err-6903.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6919.tolk b/tolk-tester/tests/invalid-typing/err-6919.tolk new file mode 100644 index 000000000..f7a988567 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6919.tolk @@ -0,0 +1,13 @@ +struct Gen { + price: coins = (1 as T) + "", +} + +fun main() { + var c: Gen = {}; + return c; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int8` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6920.tolk b/tolk-tester/tests/invalid-typing/err-6920.tolk new file mode 100644 index 000000000..dc2a5ffce --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6920.tolk @@ -0,0 +1,12 @@ +fun matchOverNotSubtypeIsNever(m: slice | cell) { + match (m) { + int => return m + 0 + }; + return 3; +} + +/** +@compilation_should_fail +@stderr `int` is not a variant of `slice | cell` +@stderr int => + */ diff --git a/tolk-tester/tests/invalid-generics-7.tolk b/tolk-tester/tests/invalid-typing/err-6921.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-7.tolk rename to tolk-tester/tests/invalid-typing/err-6921.tolk diff --git a/tolk-tester/tests/invalid-declaration-12.tolk b/tolk-tester/tests/invalid-typing/err-6925.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-12.tolk rename to tolk-tester/tests/invalid-typing/err-6925.tolk diff --git a/tolk-tester/tests/invalid-typing/err-6928.tolk b/tolk-tester/tests/invalid-typing/err-6928.tolk new file mode 100644 index 000000000..a335647ac --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6928.tolk @@ -0,0 +1,7 @@ +fun int.f(self, v: (int, int) = (0, 1+"")) { +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6930.tolk b/tolk-tester/tests/invalid-typing/err-6930.tolk new file mode 100644 index 000000000..80269c3dc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6930.tolk @@ -0,0 +1,15 @@ +type MInt = int; + +fun f1(v: int): int { return v; } + +fun main(t: int?) { + f1 as (MInt) -> MInt; + f1 as (int8) -> uint16; + + f1 as ((int | slice) -> int); +} + +/** +@compilation_should_fail +@stderr type `(int) -> int` can not be cast to `(int | slice) -> int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6937.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk new file mode 100644 index 000000000..6038ea656 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6937.tolk @@ -0,0 +1,13 @@ +fun getAnySlice() { return beginCell().endCell().beginParse(); } + +fun cantAssignSliceToBytesN() { + var c1: bytes16 = getAnySlice() as bytes16; // ok + var c2: bytes16 = getAnySlice(); +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to variable of type `bytes16` +@stderr var c2: bytes16 = getAnySlice(); +@stderr hint: use `as` operator for unsafe casting: ` as bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6942.tolk b/tolk-tester/tests/invalid-typing/err-6942.tolk new file mode 100644 index 000000000..95d0b9ad3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6942.tolk @@ -0,0 +1,17 @@ +struct Point { x: int; y: int; } + +fun Point?.methodOverNullable(self) {} + +fun cantAssignNullableObjectField(a: Point?) { + if (a != null) { + a.x = 10; // ok + } + a.methodOverNullable(); // ok + a.y = 20; +} + +/** +@compilation_should_fail +@stderr can not access field `y` of a possibly nullable object `Point?` +@stderr check it via `obj != null` or use non-null assertion `obj!` operator + */ diff --git a/tolk-tester/tests/invalid-typing/err-6948.tolk b/tolk-tester/tests/invalid-typing/err-6948.tolk new file mode 100644 index 000000000..448d07a46 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6948.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + field: T; +} + +struct AnotherWrapper { + field: T; +} + +fun main() { + var w: Wrapper = AnotherWrapper { field: 10 }; +} + +/** +@compilation_should_fail +@stderr can not assign `AnotherWrapper` to variable of type `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6981.tolk b/tolk-tester/tests/invalid-typing/err-6981.tolk new file mode 100644 index 000000000..af3bebb8c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6981.tolk @@ -0,0 +1,15 @@ +fun getTwo(): X { return 2 as X; } + +fun cantDeduceNonArgumentGeneric() { + var t1: [int] = [0]; + t1.0 = getTwo(); // ok + var t2 = createEmptyTuple(); + t2.push(0); + t2.0 = getTwo(); // deduced X = unknown +} + +/** +@compilation_should_fail +@stderr in function `getTwo` +@stderr type `int` can not be cast to `unknown` + */ diff --git a/tolk-tester/tests/invalid-typing-30.tolk b/tolk-tester/tests/invalid-typing/err-6992.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-30.tolk rename to tolk-tester/tests/invalid-typing/err-6992.tolk diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk new file mode 100644 index 000000000..ac8e9fcd5 --- /dev/null +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -0,0 +1,1504 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +fun AOrB.doMatch(self) { + __expect_inline(true); + match (self) { + A => {} + B => {} + } +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +fun uint8.doNothing(self) {} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.useAll(self) { + __expect_inline(true); + if (true) { self.method() } +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int32; + f2: int32; + f3: int32; + f4: int32; + f5: int32; + ab: A | B; +} + +fun WithEitherABEnd.checkF3(self, against: int) { + assert(self.f3 == against, 100); +} + +fun WithEitherABEnd.doMatch(self) { + __expect_inline(true); + match (self.ab) { + A => {} + B => {} + } +} + +fun rand() { return random.uint256() } + +struct Point { + x: int8; + y: int8; +} + +fun Point.assignX(mutate self, x: int) { + self.x = x; +} + +fun Point.getX(self) { + return self.x; +} + +fun Point.getXNoInline(self) { + __expect_inline(false); // due to returns in the middle + if (10>3) { + return self.x; + } + throw 123; +} + +fun Point.getCoord(self, getX: bool) { + return getX ? self.x : self.y; +} + +fun Point.getCoordWrapper(self, getX: bool) { + if (10<3) { throw 123 } + return self.getCoord(getX) +} + +fun getXOfPoint(p: Point) { + return p.x +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct WithTensor { + a1: int8; + t: (int8, int8); + a3: int8; +} + +struct WithMaybeTensor { + a1: int8; + a2: int8; + t: (int8, int8)?; +} + +struct MixedDataAndRefs { + i1: int8; + i2: bool; + a1: address; + e: (); + r1: cell; + r2: ((), cell); + i4: Point; + rm: cell?; + r4: Cell?; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct ItemState { + ownerAddress: address; + content: cell; + auction: cell?; + royaltyParams: cell; +} + +struct HasIncrementEnd { + a: int8; + inc: CounterIncrement; +} + + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(createEmptyCell()); +} + + + +fun algo1() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1"); + o.f1; + } +} + +fun algo2() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + if (rand()) { + o.f1; + } else { + o.f1 + o.f1; + } +} + +fun algo3() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + return; + } + __expect_lazy("[o] load f1"); + o.f1.doNothing(); +} + +fun algo4() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2"); + if (rand()) { + o.f1; + } else { + o.f2; + } +} + +fun algo5() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] skip (bits8), load f2, skip (bits8), load f4"); + o.f4; + o.f2; +} + +fun algo6() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits40), load f8"); + if (rand()) { + o.f8; + } + o.f2; + o.f1 * o.f1 + o.f1 + o.f2 * o.f2; +} + +fun algo7() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o; + } else { + } +} + +fun algo8() { + if (rand()) { + val o: Fields9 = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o.method(); + } else { + } +} + +fun algo9() { + val p = lazy Point3d.fromCell(createEmptyCell()); + __expect_lazy("[p] load x y z"); + if (p.x && p.y) { + p.x; + p.y; + } else { + p.z; + } +} + +fun algo10() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + p.z; + if (rand()) { + p.x; + return; + } +} + +fun algo11() { + val p = lazy Point3d.fromSlice(""); + var num = 0; + if (10 > 2) { + num += 1; + } + __expect_lazy("[p] load x"); + rand() + ? p.x as int + : p.x + num; +} + +fun algo12() { + try { + val p = lazy Point3d.fromSlice(""); + try {} + catch {} + assert(10>3, 1); + __expect_lazy("[p] skip (bits32), load y"); + p.y * p.y; + } catch { + } +} + +fun algo13() { + val o = lazy Fields9.fromSlice(""); + if (10 > 3) { + __expect_lazy("[o] load f1"); + assert(o.f1 > 0) throw 400; + } +} + +fun algo14() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + while (rand()) { + p.x; + } +} + +fun algo15() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + do { + p.y; + } while (rand()); +} + +fun algo16() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + do { + p.x; + } while (p.z); +} + +fun algo17() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + repeat(p.x) { + p.y; + } +} + +fun algo18() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits16), load f5 f6"); + if (o.f1) { + if (o.f2) { + throw o.f6 + } + } + else { o.f5 } +} + +fun algo19() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + var p2 = p; + return (p2.x, p2.y); +} + +fun algo20() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] skip (bits8), load p"); + return c.p.x; +} + +fun algo21() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p"); + if (c.f1 > 0) { + return c.p.y; + } + throw 0; +} + +fun algo22() { + var w = lazy WithTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load t"); + w.t.0 = 10; +} + +fun algo23() { + var w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load a2 t"); + if (w.t == null) { + return w.a2; + } + return w.t.0; +} + +fun algo24() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno"); + return w.seqno as int; + } catch (excno) { + return excno; + } +} + +fun algo25() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno as int; + } catch (excno) { + if (excno > 0) { + return w.seqno as int; + } + return excno; + } +} + +fun algo26() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno + 1; + } catch (excno) { + } + return w.seqno - 1; +} + +fun algo27() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return w.seqno + 1; + } catch (excno) { + } + if (10 > 3) { + return w.walletId as int; + } + return -1; +} + +fun algo28() { + val w = lazy WalletStorage.fromSlice(""); + try { + try { + return 1; + } catch { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + return w.walletId as int; + w.seqno; + } + } catch (excno) { + return -1; + } +} + +fun algo29() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } +} + +fun algo30() { + val w = lazy WalletStorage.fromSlice(""); + if (true) { + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } + } +} + +fun algo31() { + val w = lazy WalletStorage.fromSlice(""); + if (false) {} + else { + __expect_lazy("[w] skip (bits65), load publicKey"); + while (true) { + w.publicKey + } + } +} + +fun algo32() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] load isSignatureAllowed"); + match (true) { + true => { w.isSignatureAllowed } + false => {} + } +} + +fun algo33() { + val p = lazy ComplexWithPoint.fromSlice(""); + try { if(true) {} else { + __expect_lazy("[p] skip (bits8), load p"); + p.p.x; + }} catch{} +} + +fun algo34() { + val p = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p] skip (bits8), load p f3"); + if (p.p.x) { } + else { p.f3 } +} + +fun algo35() { + var p = lazy ComplexWithPoint.fromSlice(""); + try { + __expect_lazy("[p] load f1 p"); + if (p.p.x) { } + else { p.p = {x:1, y:2} } + if (true) { + try {} catch (excno) { match (excno + p.f1) { } } + } + } catch {} +} + +fun algo36() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p"); + try { + __expect_lazy("[p2] load f1 p f3"); + match (true) { + true => { p2.f3 } + else => p2.p.x ? p2.f1 : p1.p.x + } + } catch { p1.p.y -= 1 } +} + +fun algo37() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + try { + if (10>3) { + __expect_lazy("[p2] load f1 p f3"); + var p1 = p2; + } + } catch { + __expect_lazy("[p1] skip (bits8), load p"); + p1.p.y -= 1 + } +} + +fun algo38() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p f3"); + try { + if (10>3) { + __expect_lazy("[p2] skip (bits24), load f3"); + p1.f3 = p2.f3; + } + } catch { + p1.p.y -= 1 + } +} + +fun algo39() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + return p.getX(); +} + +@method_id(140) +fun algo40() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoord(false); +} + +@method_id(141) +fun algo41(getX: bool) { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoordWrapper(getX); +} + +fun algo42() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // calling a mutating method — not available for lazy + p.assignX(10); +} + +fun algo43() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not a method — not available for lazy (not implemented) + return getXOfPoint(p); +} + +fun algo44() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not an inlined method + return p.getXNoInline(); +} + +fun algo46() { + var w = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[w] skip (bits64), load f3, skip (bits64), load ab"); + w.checkF3(0); + w.doMatch(); +} + +fun algo47() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + f.useAll(); +} + +fun algo48() { + var p = lazy Point.fromSlice(""); + contract.getData(); + __expect_lazy("[p] load x y"); + p.forceLoadLazyObject(); +} + +fun algo49(forceLoad: bool) { + val l = lazy Fields9.fromSlice(""); + if (forceLoad) { + __expect_lazy("[l] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + try { l.forceLoadLazyObject() } + catch { l.f2 } + } +} + +fun algo50() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] skip (bits8) (int8?) (bits9) (int8 | bool) (address) (bits608), load f11"); + st.f11; +} + +fun algo51() { + var w1 = lazy MixedDataAndRefs.fromSlice(""); + var w2 = lazy MixedDataAndRefs.fromSlice(""); + var w3 = lazy MixedDataAndRefs.fromSlice(""); + var w4 = lazy MixedDataAndRefs.fromSlice(""); + var w5 = lazy MixedDataAndRefs.fromSlice(""); + var w6 = lazy MixedDataAndRefs.fromSlice(""); + + __expect_lazy("[w1] load r1"); + w1.r1; + __expect_lazy("[w2] skip (cell), load r2"); + w2.r2; + __expect_lazy("[w3] skip (bits9) (address) (cell) (((), cell)) (bits16), load rm"); + w3.rm; + __expect_lazy("[w4] skip (bits9) (address), load i4"); + w4.i4; + __expect_lazy("[w5] skip (bits8), load i2, skip (address), load e r1 i4"); + w5.i2; w5.e; w5.r1; w5.i4; +} + +fun algo52(cc: Cell?) { + val itemState = lazy cc!.load(); + __expect_lazy("[itemState] skip (address) (cell) (cell?), load royaltyParams"); + return itemState.royaltyParams; +} + +fun algo53() { + val i = lazy ItemState.fromSlice(""); + __expect_lazy("[i] load content"); + return i.content.beginParse(); +} + + +fun algo100() { + var o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1, skip (bits8), load a3"); + o.a1; + __expect_lazy(""); + o.a3 + } + B => { + __expect_lazy("[o] load b1 b2"); + o.b2; + __expect_lazy(""); + o.b1 + } + } +} + +fun algo101() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab, skip (bits32), load f5"); + o.f5; + o.ab; +} + +fun algo102() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5 ab"); + o.f5; + o.ab; +} + +fun algo103() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo104() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo105() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3, skip (bits64), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, o.f3); + // __expect_lazy("[o.ab] load a1 a2"); + return o.ab.a1 + o.ab.a2; + } + B => { + // __expect_lazy("[o.ab] load b1 b2 b3"); + val cp = o.ab; + return cp.b1 + cp.b2; + } + } +} + +fun algo106() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits96), load f4, skip (bits32), lazy match ab"); + if (rand()) { + o.f4; + } else { + match (o.ab) { A => {} B => { + o.ab.b2; + } } + o.f4; + } +} + +fun algo107() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + if (o.ab is B) {} + match (o.ab) {} +} + +fun algo108(): int7 | int8 { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + return match (o.ab) { + A => { + // __expect_lazy("[o.ab] load a1"); + return o.ab.a1; + } + B => { + // __expect_lazy("[o.ab] skip b1, load b2"); + return o.ab.b2; + } + } +} + +fun algo109(): int { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4, skip (bits32), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, 100); + // __expect_lazy("[o.ab] load a1"); + return o.f1 + o.ab.a1; + } + B => { + return o.f2 + o.f4; + } + } +} + +fun algo110() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => p.y, + 2 => p.z, + } +} + +fun algo111() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => 1, + else => { var x = p.y + p.y } + } + p.z; +} + +fun algo112() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits8), load f4 f5 f6"); + match (o.f2) { + uint8 => { + o.f1 + match(o.f4) { 1 => o.f4, else => o.f5 } + } + } + o.f6; +} + +fun algo113() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + match (o.f1) { + uint8 => { + debug.print(o.f1); + } + } + o.f1; +} + +type PointOrPoint3d = Point | Point3d; + +fun algo114() { + val o = lazy PointOrPoint3d.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + Point => { + __expect_lazy("[o] load x y"); + return o.x + o.y; + } + Point3d => { + __expect_lazy("[o] load x y z"); + var p = o; + return p.x + p.y; + } + } +} + +fun algo115() { + val o = lazy WithEitherABMiddle.fromSlice(""); + assert(10>3, 1); + __expect_lazy("[o] load f1 f2 ab f4"); + o.f1 + o.f2; + match (o.ab) { + A => { o.f1 + o.ab.a1 } + B => { o.f1 + o.f2 + o.ab.b1 } + } + o.f4; +} + +fun algo116() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] load f1 f2 ab f4 f5"); + o.f1 + o.f2; + match (o.ab) { + A => { return o.f4 } + B => throw o.f5 + } +} + +fun algo117() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4 f5, lazy match ab"); + o.f1 + o.f2; + match (o.ab) { + A => return o.f4, + B => throw o.f5 + } +} + +fun algo118() { + val o = lazy WithEitherABEnd.fromSlice(""); + if (10>3) { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } + throw 123; +} + +fun algo119() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { + return o.ab.a2 + o.f2; + } + B => { + assert(o.ab.b1) throw o.f1; + } + } + o.f1; + return o.f2 as int; +} + +fun algo120() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => return o.ab.a2 * 1, + B => return o.ab.b1 << 0, + } + o.f1; +} + +fun algo121() { + val o1 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o1] load f1 f2 f3 f4 f5, lazy match ab"); + match (o1.ab) { + A => { (o1.f1, o1.f2, o1.f3, o1.f4, o1.f5) } + B => {} + } + val o2 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o2] load f1 f2 f3 f4 f5 ab"); + match (o2.ab) { + A => { (o2) } + B => {} + } +} + + +type AOrWithEitherABEnd = A | WithEitherABEnd; + +fun algo122() { + val o = lazy AOrWithEitherABEnd.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1"); + o.a1; + } + WithEitherABEnd => { + __expect_lazy("[o] skip (bits160), load ab"); // not lazy match, nesting not supported + match (o.ab) { + A => { o.ab.a1 } + B => {} + } + } + } +} + +fun algo123() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + __expect_lazy("[o] load a1"); + throw o.a1; + } + B => { + __expect_lazy("[o] load b1 b2"); + assert(10>o.b2) throw 100; + return o.b1; + } + } +} + +fun algo124() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + assert(10>3, 100); + assert(10>3, 100); + __expect_lazy("[o] load a1 a2 a3"); + throw (100, o.getA1()); + } + B => { + assert(10>3, 100); + __expect_lazy("[o] load b1 b2 b3"); + return o is B; + } + } +} + +fun algo127() { + val w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits16), load t"); + match (w.t) { + null => {} + (int8, int8) => {} + } +} + +fun algo128() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + match (o) { A => {} } + } + B => {} + } +} + +fun algo129() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5, lazy match ab"); + match (o.ab) { + A => {} + B => { + match (o.ab) { B => { o.f5 } } + } + } +} + +fun algo130() { + var o = lazy AOrB.fromSlice(""); + try { + return 1; + } catch (excno) { + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1"); + return o.a1 + excno; + } + B => { + __expect_lazy("[o] load b1 b2 b3"); + if (o.b1 > 0) { o = B { b1: 1, b2: 2, b3: 3 } } + return o.b2 as int; + } + } + } +} + +fun algo131() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (10 > 3) { return -1; } + else { + if (10 > 3) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 0; + } + } + } + } +} + +fun algo132(): int { + val o = lazy AOrB.fromSlice(""); + match (o) { + B => { + __expect_lazy("[o] load b1 b2"); + try { + return o.b1; + } catch { + return o.b2; + } + } + A => { + __expect_lazy("[o] skip (bits7), load a2 a3"); + try { + return o.a3; + } catch { + throw 123; + } + return o.a2; + } + } +} + +fun algo133(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try {} + catch { + try { + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => { } + B => { o.f1 } + } + } catch { throw 123 } + } +} + +fun algo134(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try { try { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + try { + match (o.ab) { + A => { } + B => { } + } + } catch { throw o.f1 + o.f2 } + } catch {} } catch {} +} + +fun algo135() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + o.a1; + o.forceLoadLazyObject().assertEnd() + } + B => { + __expect_lazy("[o] load b1 b2"); + if (o.b1) { + o.b2; + } + } + } +} + +fun algo136() { + while (true) { + val obj = lazy AOrB.fromSlice(""); + __expect_lazy("[obj] lazy match"); + match (obj) { + A => { __expect_lazy("[obj] load a1"); obj.a1 } + B => { __expect_lazy("[obj] skip (bits7), load b2"); obj.b2 } + } + } +} + + +fun algo150() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + __expect_lazy("[o] skip (bits160), load ab"); + while (!found) { + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo151() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + if (!found) { + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo152() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 + match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo153() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 && match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo224(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits32), load f2, skip (bits32), load f4 f5 ab"); + match (o.ab) { + A => {} + B => { o.f2; } + } + match (o.ab) { + A => {} + B => { if (rand()) { o.f4; } return; } + } + if (o.ab is A) { + if (rand()) { + o.f5; + } + } +} + +fun algo225() { + __expect_lazy("[c] lazy match"); + match (val c = lazy MsgFullCounter.fromSlice("")) { + CounterIncrement => { + var d = lazy WalletStorage.fromSlice(""); + __expect_lazy("[d] load (bits1) seqno, save immutable (tail)"); + d.seqno += 1; + d.toCell(); + __expect_lazy("[c] load byValue"); + 1 + c.byValue; + } + CounterDecrement => { + val c = lazy MsgFullCounter.fromSlice(""); + __expect_lazy("[c] lazy match"); + match (c) { + CounterIncrement => {} + CounterDecrement => {} + CounterDecrementBy1 => {} + CounterReset => { + assert(10>3, 101); + __expect_lazy("[c] load initialValue"); + while (10>3) { + if (10>3) { c.initialValue } + } + } + } + } + CounterDecrementBy1 => { + } + CounterReset => { + if (10 > 3) { + __expect_lazy("[c] load initialValue"); + c.initialValue + } + } + else => throw 123, // `else` is allowed in a lazy `match` + } +} + +fun algo226() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits1), load seqno"); + return st.seqno; +} + +fun algo250() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + uint8 => {} + } +} + +fun algo251() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + 0 => 1, + 1 => 0, + else => throw 0, + } +} + +fun algo252() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + match (p.y) { + int8 => {} + } +} + +fun algo253() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + match (p.y) { + 0 => p.x, + 1 => 0, + else => throw 0, + } +} + +fun algo254() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + return 1; + } + } +} + +fun algo255() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + __expect_lazy("[p] load x"); + return p.x; + } + else => throw 123 + } +} + +fun algo256() { + var p = lazy Point.fromSlice(""); + match (p) { + Point => { + __expect_lazy("[p] load x"); + match (p.x) { int8 => {} } + } + } +} + +fun algo257(): int { + val k = lazy HasIncrementEnd.fromSlice(""); + __expect_lazy("[k] skip (bits8), load inc"); + match (k.inc) { + CounterIncrement => { + __expect_lazy(""); + return k.inc.byValue; + } + } +} + + +fun algo300() { + val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304")); + __expect_lazy("[p] lazy match"); + match (p) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + debug.print(p.a2); + } + B => { + __expect_lazy("[p] load b1 b2"); + debug.print(p.b1 + p.b2); + } + } +} + +@method_id(301) +fun algo301() { + __expect_lazy("[p] lazy match"); + match (val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304"))) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + return p.a2 as int; + } + B => { + __expect_lazy("[p] load b1 b2"); + return p.b1 + p.b2; + } + } +} + +fun algo302() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy Point.fromCell(c); + __expect_lazy("[o] load x y"); + o.x; + o.y; + return o; +} + +@method_id(303) +fun algo303() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy c.load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(304) +fun algo304() { + var o = lazy Point{x:1,y:2}.toCell().load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(305) +fun algo305() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + assert(10>3, 100); + __expect_lazy("[p] load x y"); + { + assert(p.x == 1, 100); + assert(p.y == 2, 100); + } + return p; +} + +fun algo306() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail)"); + st.seqno += 1; + return st.toCell(); +} + +fun algo307() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.assignX(10); + c.toCell(); +} + +fun algo308() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.x = 10; + c.toCell(); +} + +fun algo309() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p f3"); // object used for writing and toCell, load all for safety + c = { f1: 10, p: {x:0,y:0}, f3: c.f3 }; + c.toCell(); +} + +fun algo310() { + var c = lazy ComplexWithPoint.fromSlice(""); + if (true) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + debug.print(c.p.y += 2); + c.toCell(); + } +} + +fun algo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = true; + st.extensions = null; + return st.toCell(); +} + +fun algo312() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) (int8 | bool) (address) f8, save immutable (tail)"); + if (rand()) { + st.f8 += 1; + } + st.toCell(); +} + +fun algo313() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + if (rand()) { + __expect_lazy("[st] load (bits8) (int8?) (bits9) f5, save immutable (tail), skip (int8 | bool) (address) (bits8), load f9"); + st.f5 = createEmptyCell(); + st.f9; + st.toCell(); + } +} + + +fun main() { +} + +/** +@testcase | 140 | | 2 +@testcase | 141 | -1 | 1 +@testcase | 301 | | 17 +@testcase | 303 | | 1 2 +@testcase | 304 | | 1 2 +@testcase | 305 | | 1 2 + +@fif_codegen +""" + algo40() PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + 8 PLDI + }> +""" + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk new file mode 100644 index 000000000..f686fe67c --- /dev/null +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -0,0 +1,2034 @@ +import "@stdlib/tvm-dicts.tolk" + +struct A { a1: int7; a2: uint8; a3: uint8; } +struct B { b1: int7; b2: uint8; b3: uint8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.getF2(self) { return self.f2 } +fun Fields9.getF5(self) { return self.f5 } + +struct Small { s1: int7; } +struct Big { b1: int7; b2: int8; b3: int8; } +type SmallOrBig = Small | Big; + +struct StructF1F2AB { + f1: int8; + f2: int8; + ab: A | B; +} + +struct StructF1SBF3 { + f1: int8; + sb: Small | Big; + f3: int8; +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int8; + f2: int8; + f3: int8; + f4: int8; + f5: int8; + ab: A | B; +} + +struct Point { + x: int8; + y: int8; +} + +fun Point.getX(self) { + return self.x +} + +fun Point.makeCell(self) { + return self.toCell() +} + +fun Point.makeCellWrapped(self) { + val cc = self.makeCell(); + self.x; + return cc +} + +fun Point.incXAndMakeCell(mutate self) { + __expect_inline(true); + self.x += 1; + return self.toCell() +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +type PointOrPoint3d = Point | Point3d; + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +fun WalletStorage.save(self) { + contract.setData(self.toCell()) +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct Inner1 { i1: int8; i2: int8; } +struct Inner2 { i1: int8; i2: int8; } +type Inner2Alias = Inner2; +struct Outer { + inner1: Inner1; + inner2: Inner2Alias; + wa: StructF1F2AB; +} + +struct WithMaybe { + f1: int8; + big: Big?; + f3: int2; +} + +struct WithRefs { + r1: cell?; + f2: int8; + r3: Cell; + f4: int8; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct(0x12345678) Counter32Increment { byValue: int32; } +struct(0x23456789) Counter32DecrementBy1 {} +struct(0x34567890) Counter32Reset { initial: uint32; f2: int8; f3: int8; f4: int8; f5: int8; } +type MyMsg32 = Counter32Increment | Counter32DecrementBy1 | Counter32Reset; + +struct AutoP1 { f1: int8; f2: int8; } +struct AutoP2 { f1: int8; f2: int8; f3: int8; } +struct AutoP3; +type AutoPrefixedBin2 = AutoP1 | AutoP2 | AutoP3; // 00 / 01 / 10 + +struct(0b0111) ShortP1 { f1: int8; } +struct(0b0100) ShortP2 { f1: int8; f2: int8; } +type ShortPrefixed = ShortP1 | ShortP2; + +struct(0b01) NotEqPrefixedS1 { f: uint6; } +struct(0b0001) NotEqPrefixedS2 { f: uint4; } +struct(0b101) NotEqPrefixedS3 { f: int5; } +type NotEqPrefixed = NotEqPrefixedS1 | NotEqPrefixedS2 | NotEqPrefixedS3; + +struct WithBitsField { + bin16: bits16; + bin8: bits8; +} + +struct WithOneRestField { + i32: int32; + rest: RemainingBitsAndRefs; +} + +struct VestingStorage { + vestingParameters: Cell<(uint64, int32)>; + seqno: uint32; + subwalletId: uint32; + publicKey: uint256; + whitelist: dict; +} + +struct OneIntOneRef { + z: cell? = null; + o: cell; + e: () = (); + i: int32 = 0; + r: cell; +} + +struct WithVariadicInts { + i1: varuint32; + i2: varuint32; +} + +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + +struct WithN5 { n: uint5 } +type U111 = int8 | int16 | WithN5 | bits100 +struct WithU111 { + a: U111 + b: U111? + bit: bool +} +fun WithU111.getSlice_a8_bnull(): slice asm "b{000000000101} PUSHSLICE" +fun WithU111.getSlice_a16_b8(): slice asm "b{010000000000000001100000000011} PUSHSLICE" +fun WithU111.getSlice_a8_b5(): slice asm "b{0000000001110111111} PUSHSLICE" +fun WithU111.getSlice_a8_b5_nobit(): slice asm "b{000000000111011111} PUSHSLICE" + + +@noinline +fun contract.getFakeData(seqno: int): Cell { + return WalletStorage { + isSignatureAllowed: true, + seqno: seqno, + walletId: 999, + publicKey: 1 << 100, + extensions: null, + }.toCell() +} + +fun getPointCell(x: int, y: int): Cell { + val p: Point = {x,y}; + return p.toCell(); +} + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(contract.getFakeData(777)); +} + +@method_id(101) +fun demo101() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x"); + return p.x; +} + +@method_id(102) +fun demo102(takeX: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result: int; + __expect_lazy("[p] load x y"); + if (takeX) { + result = p.x; + } else { + result = p.y; + } + return result; +} + +@method_id(103) +fun demo103() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] skip (bits8), load y"); + return p.y * p.y * p.y; +} + +@method_id(104) +fun demo104() { + var p2 = lazy Point.fromSlice(stringHexToSlice("0A14")); + __expect_lazy("[p2] load x y"); + assert(p2.x < 100) throw 100; + assert(p2.y < 100) throw 100; + + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01020304")); + __expect_lazy("[c] load f1, skip (bits16), load f3"); + return (c.f1, 0 + c.f3); +} + +@method_id(105) +fun demo105(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + __expect_lazy("[c] load f1, skip (Point78ForMaybe?), load f3"); + return (c.f1, c.f3); +} + +@method_id(106) +fun demo106(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + var x: int? = null; + __expect_lazy("[c] skip (bits8), load p"); + if (c.p != null) { + x = c.p.x; + } + return (c.p == null, x); +} + +@method_id(107) +fun demo107() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + return p; +} + +@method_id(108) +fun demo108() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + { + assert(o.f3 == 3) throw 100; + assert(o.f9 == 9) throw 100; + } + + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + p.y = 15; + return (p, o); +} + +@method_id(109) +fun demo109() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] skip (bits65), load publicKey"); + return st.publicKey; +} + +@method_id(110) +fun demo110(s: slice) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter7Increment => { + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(111) +fun demo111(s: slice, x: int) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => { + assert(x >= 0, 123); + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + assert(x >= 1, 456); + assert(t.size() == 0, 789); + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(112) +fun demo112(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterIncrement => curX += msg.byValue, + CounterDecrement => curX -= msg.byValue, + CounterDecrementBy1 => curX = -1, + CounterReset => curX = msg.initialValue, + } + return curX; +} + +@method_id(113) +fun demo113(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int = match (msg) { + CounterIncrement => curX + msg.byValue, + CounterDecrement => curX - msg.byValue, + CounterDecrementBy1 => -1, + CounterReset => msg.initialValue, + }; + return newX; +} + +@method_id(114) +fun demo114(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int | [int] = match (msg) { + CounterDecrementBy1 => -1, + CounterReset => [msg.initialValue as int], + CounterDecrement => [curX - msg.byValue], + CounterIncrement => curX + msg.byValue, + }; + return newX; +} + +@method_id(115) +fun demo115(s: slice): int8 | bool { + var msg = lazy MsgEitherCounter.fromSlice(s); + return match (msg) { + Counter7Decrement => msg.byValue < 5, + Counter7Increment => msg.byValue as int8, + } +} + +@method_id(118) +fun demo118(s: slice) { + match (var msg = lazy MsgEitherCounter.fromSlice(s)) { + Counter7Increment => return +msg.byValue, + Counter7Decrement => { + assert(s.remainingRefsCount() == 0, 456); + __expect_lazy("[msg] load byValue"); + return -msg.byValue + } + } +} + +@method_id(119) +fun demo119(s: slice) { + var o = lazy SmallOrBig.fromSlice(s); + var s1 = -1; + var b1 = -1; + match (o) { + Small => { + __expect_lazy("[o] load s1"); + s1 = o.s1; + } + Big => { + __expect_lazy("[o] skip (bits7), load b2"); + b1 = o.b2; + } + } + return (s1, b1); +} + +@method_id(120) +fun demo120(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] load f1, skip (bits8), lazy match c"); + match (msg.c) { + Counter7Increment => { t.push(+msg.c.byValue) } + Counter7Decrement => { t.push(-msg.c.byValue) } + } + t.push(msg.f1); + return t; +} + +@method_id(121) +fun demo121() { + var c = FieldsAndEitherCounter{f1:1, f2:2, c:Counter7Decrement{byValue:20}}; + var o = lazy FieldsAndEitherCounter.fromCell(c.toCell()); + __expect_lazy("[o] load f1 f2 c"); // load c, not lazy match + return (o.c is Counter7Increment) ? o.f1 + o.c.byValue : o.f2 + o.c.byValue; +} + +@method_id(122) +fun demo122(s: slice): int | slice { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1, skip (bits8), lazy match c"); + return match(o.c) { + Counter7Decrement => -o.c.byValue, + Counter7Increment => o.f1 + o.c.byValue, + } +} + +@method_id(123) +fun demo123(s: slice) { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1 f2 c"); // not lazy match, used in a complex expression + return o.f1 + match (o.c) { + Counter7Increment => o.f1 as int, + Counter7Decrement => o.f2 * o.f2, + } +} + +@method_id(124) +fun demo124(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] skip (bits8), load f2, lazy match c"); + match (msg.c) { + Counter7Increment => { + var mm = msg.c; + t.push(+mm.byValue) + } + Counter7Decrement => { + try {} + catch {} + var mm = msg.c; + t.push(-mm.byValue); + t.push(msg.f2); + } + } + return t; +} + +@method_id(125) +fun demo125() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + var (a1, a3, b2) = (-1,-1,-1); + __expect_lazy("[o] load f1 f2 ab"); + match (o.ab) { + A => { + a1 = o.ab.a1; + a3 = o.ab.a3; + } + B => { + b2 = o.ab.b2; + } + } + return (o, a1, a3, b2); +} + +@method_id(126) +fun demo126(s: slice) { + var o = lazy StructF1SBF3.fromSlice(s); + var (s1, b2) = (-1, -1); + __expect_lazy("[o] load f1 sb f3"); + match (o.sb) { + Small => { + s1 = o.sb.s1; + } + Big => { + b2 = o.sb.b2; + } + } + return (o, s1, b2); +} + +@method_id(127) +fun demo127() { + val o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] load inner1, skip (bits16), load wa"); + var inner1_i1 = o.inner1.i1; + var inner1_i2 = o.inner1.i2; + + var wa_is_A: bool; + var wa_last: int; + match (o.wa.ab) { + A => { wa_is_A = true; __expect_lazy(""); wa_last = o.wa.ab.a3; } + B => { wa_is_A = false; __expect_lazy(""); wa_last = o.wa.ab.b3; } + } + return (inner1_i1, inner1_i2, wa_is_A, wa_last); +} + +@method_id(128) +fun demo128(s: slice) { + var o = lazy WithMaybe.fromSlice(s); + __expect_lazy("[o] load f1 big"); + assert (o.f1 > 0) throw 123; + var isnull = true; + var b3 = -1; + match (o.big) { + null => { + debug.printString("big is null"); + } + Big => { + isnull = false; + assert(o.big.b1 != 0, 123); + b3 = o.big.b3; + } + } + return (isnull, b3); +} + +@method_id(129) +fun demo129() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203")); + var (a1, wasA) = (-1, false); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + a1 = o.getA1(); + wasA = true; + } + B => { + __expect_lazy("[o] skip (bits7), load b2"); + a1 = o.b2; + } + } + return (wasA, a1); +} + +@method_id(130) +fun demo130() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits33), load walletId"); + return st.walletId; +} + +@method_id(131) +fun demo131() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits32), load publicKey"); + return (st.seqno, st.publicKey >> 100); +} + +@method_id(132) +fun demo132(incSeqno: bool) { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits288), load extensions"); + if (incSeqno) { + return (st.seqno + 1, st.extensions == null); + } else { + return (st.seqno as int, st.extensions == null); + } +} + +@method_id(133) +fun demo133() { + var extensions = createEmptyDict(); + extensions.iDictSet(32, 8, stringHexToSlice("12")); + var wCell = WalletStorage { + isSignatureAllowed: true, + seqno: 0, + walletId: 123, + publicKey: 1 << 88, + extensions, + }.toCell(); + if (10 > 3) { + var st = lazy wCell.load(); + __expect_lazy("[st] skip (bits321), load extensions"); + while (!false) { + assert(st.extensions != null, 123); + return st.extensions.iDictGet(32, 8).0!.loadUint(8); + } + } else { + throw 0; + } +} + +@method_id(134) +fun demo134() { + var cc = lazy CounterIncrement.fromSlice(stringHexToSlice("010f"), {throwIfOpcodeDoesNotMatch: 134}); + return cc.byValue; +} + +@method_id(135) +fun demo135(skip: bool) { + var cc = lazy WithBitsField.fromSlice(stringHexToSlice("123456")); + __expect_lazy("[cc] load bin16, save immutable (tail)"); + cc.bin16 = stringHexToSlice("00000000") as bits16; + try { + return cc.toCell({ + skipBitsNValidation: skip + }).beginParse().remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(136) +fun demo136(skip: bool): int { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + if (skip) { + return -1; + } else { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } +} + +@method_id(137) +fun demo137(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] load x"); + return p.x; // will be ok even if s doesn't contain y + } catch (excno) { + return 100 + excno; + } +} + +@method_id(138) +fun demo138(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } catch (excno) { + return 100 + excno; + } +} + +@method_id(139) +fun demo139(s: slice, skip: bool): int { + var msg = lazy MsgFullCounter.fromSlice(s); + match (msg) { + CounterIncrement => { + try { + if (s.isEmpty()) { return 100; } + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } catch (excno) { return excno } + } + CounterReset => throw 123, + CounterDecrementBy1 => throw 456, + CounterDecrement => { + if (!skip) { + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } + return 100; + } + } +} + +@method_id(140) +fun demo140(getF4: bool) { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040580FF00")); + var result: int; + __expect_lazy("[o] skip (bits24), load f4, skip (bits8), lazy match ab"); + if (getF4) { + result = o.f4; + } else { + match (o.ab) { A => throw 123, B => { + result = o.ab.b2; + } } + if (o.f4 < 0) { result *= 100; } + } + return result; +} + +@method_id(141) +fun demo141(eval: bool): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040500FF00")); + if (eval) { + try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } catch(excno) { return excno * 100 } + } + throw 123; +} + +@method_id(142) +fun demo142(s: slice, ignore: bool) { + val o = lazy AOrB.fromSlice(s); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (ignore) { return -1; } + else { + if (ignore) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 1; + } + } + } + } +} + +@method_id(143) +fun demo143(): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("0809000000")); // ab missing + try { try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + try { + match (o.ab) { + A => { return 1 } + B => { return 2 } + } + } catch { throw o.f1 + o.f2 } + } catch(excno) { return excno } } catch { return -1 } +} + +@method_id(144) +fun demo144(dummy: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result = -1; + if (!dummy) { + __expect_lazy("[p] load x"); + result = p.getX(); + } + return result; +} + +@method_id(145) +fun demo145(getF5: bool): int { + var p = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + try { + __expect_lazy("[p] skip (bits8), load f2, skip (bits16), load f5"); + if (getF5) { + return p.getF5(); + } + return 1 * p.getF2(); + } catch { throw 123 } +} + +@method_id(146) +fun demo146() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); // no mandatory ref exists, but since we don't access it, it doesn't fail + __expect_lazy("[p] load r1"); + return p.r1 == null; +} + +@method_id(147) +fun demo147() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); + if (false) {} else { + try { + __expect_lazy("[p] skip (cell?) (bits8), load f4"); + return (true, p.f4 as int); // even though `r3` (Cell) doesn't exist, it's ignored to reach `f4` + } catch (excno) { + return (false, excno); + }} +} + +@method_id(148) +fun demo148() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203FF")); + match (o) { + A => { + var bits: int; + __expect_lazy("[o] load a1 a2 a3"); + bits = o.forceLoadLazyObject().remainingBitsCount(); + return bits; + } + B => throw 123 + } +} + +@method_id(149) +fun demo149() { + var o = lazy AOrB.fromSlice(stringHexToSlice("8F02")); // corrupted, too small + match (o) { + A => throw 123, + B => { + try { + __expect_lazy("[o] load b1 b2 b3"); + o.forceLoadLazyObject().assertEnd(); + return o.b1 << 8 + } catch (excno) { + return excno + } + } + } +} + +@method_id(150) +fun demo150() { + val o = lazy WithOneRestField.fromSlice(stringHexToSlice("00000001FFFF")); + return (o.i32, o.rest.remainingBitsCount()); +} + +@method_id(151) +fun demo151() { + val c = VestingStorage { + whitelist: createEmptyDict(), + subwalletId: 123, + publicKey: 456, + vestingParameters: (1 as uint64, 2 as int32).toCell(), + seqno: 10, + }.toCell(); + var st = lazy c.load(); + __expect_lazy("[st] skip (Cell<(uint64, int32)>) (bits320), load whitelist"); + return st.whitelist == null; // need to skip prev bits, because dict requires 1 bit +} + +@method_id(152) +fun demo152() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?) (cell), load r"); + return st.r.beginParse().remainingBitsCount(); +} + +@method_id(153) +fun demo153() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?), load i"); + return st.i; +} + +global gTup: tuple; + +@noinline +fun pushToGlobalTup(v: T) { gTup.push(v.__toTuple()); } + +@method_id(154) +fun demo154() { + gTup = createEmptyTuple(); + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + pushToGlobalTup(p); + return gTup; +} + +fun inlinedFunctionWithLazy(s: slice) { + __expect_inline(true); + val p = lazy Point.fromSlice(s); + assert(p.y >= 0, 555); +} + +@method_id(155) +fun demo155() { + inlinedFunctionWithLazy(stringHexToSlice("0102")); + inlinedFunctionWithLazy(stringHexToSlice("0304")); + try { + inlinedFunctionWithLazy(stringHexToSlice("03FF")); + } catch (excno) { return excno; } + return -1; +} + +@method_id(156) +fun demo156(s: slice): int { + val d = lazy CounterIncrement.fromSlice(s); + __expect_lazy("[d] lazy match"); + match (d) { + CounterIncrement => { + __expect_lazy("[d] load byValue"); + return d.byValue; + } + else => { + return -100; + } + } +} + +@method_id(157) +fun demo157() { + var c: Cell = WithVariadicInts { i1: 1 << 100, i2: 1 << 200 }.toCell(); + val d = lazy c.load(); + return d.i2 == (1 << 200); +} + +@method_id(158) +fun demo158() { + val m1 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("01FF")); + __expect_lazy("[m1] skip (bits8), load g"); + m1.g; // modifies gModByCustom by a custom unpack function + val after1 = gModByCustom; + val m2 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("010A02")); + __expect_lazy("[m2] skip (bits8) (MagicGlobalModifier), load n"); + return (m2.n, gModByCustom, after1); // also modifies by skipping (unpack called) +} + +@method_id(159) +fun demo159() { + val p1 = lazy WithU111.fromSlice(WithU111.getSlice_a8_bnull()); + val p2 = lazy WithU111.fromSlice(WithU111.getSlice_a16_b8()); + val p3 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5()); + val p4 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5_nobit()); + + val p1_bit = p1.bit; + val p2_bit = p2.bit; + val p3_bit = p3.bit; + val p4_b = p4.b; + + return (p1_bit, p2_bit, (p3.b is WithN5) ? p3.b.n as int : 777, p3_bit, (p4_b is WithN5) ? p4_b.n as int: 777); +} + + +@method_id(200) +fun demo200() { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + var b = beginCell(); + if (10>3) { + __expect_lazy("[st] load isSignatureAllowed seqno walletId publicKey extensions"); + assert(st.isSignatureAllowed, 123); + b.storeAny(st); + var st2 = lazy WalletStorage.fromCell(b.toCell()); + __expect_lazy("[st2] load isSignatureAllowed"); + if (st2.isSignatureAllowed) { + return 1; + } + } + throw 123; +} + +@method_id(201) +fun demo201() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var c1 = p.toCell(); + var x = p.x; + var c2 = p.toCell(); + var y = p.y; + var c3 = p.toCell(); + assert(c1.beginParse().bitsEqual(c2.beginParse()), 100); + assert(c1.beginParse().bitsEqual(c3.beginParse()), 100); + return (x, y, p is Point); +} + +@method_id(202) +fun demo202() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2, save immutable (tail), skip (bits24), load f6"); + if (o.f1 == 1) { + o.f2 = 127; + } + var f6 = o.f6; + var o2 = lazy o.toCell().load(); + __expect_lazy("[o2] skip (bits8), load f2"); + return (o.f2, o2.f2, f6); +} + +@method_id(203) +fun demo203() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + if (o.f1 == 1) { + return o.toCell().load(); + } + throw 123; +} + +@method_id(204) +fun demo204() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4, save immutable (tail)"); + if (o.f1 != 1) { + return o.toCell().load(); + } else { + o.f1 = 10; + } + if (o.f2 != 2) { + return o.toCell().load(); + } else { + o.f2 = 20; + } + if (o.f3 != 3) { + return o.toCell().load(); + } else { + o.f4 = 40; + } + return o.toCell().load(); +} + +@method_id(205) +fun demo205() { + var o = lazy WithMaybe.fromSlice(stringHexToSlice("551_")); + __expect_lazy("[o] save immutable (tail), skip (bits8), load big"); + match (o.big) { null => {} Big => {} } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("551_")); +} + +@method_id(206) +fun demo206() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + o.method(); + return (o.toCell() is Cell, o.toCell().beginParse().remainingBitsCount()); +} + +@method_id(207) +fun demo207() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: null, + f2: 123, + r3: getPointCell(10, 20), + f4: 55, + }.toCell()); + o.r1 = getPointCell(60, 70); + o.f2 = 90; + return o.toCell(); +} + +@method_id(208) +fun demo208(s: slice) { + var o = lazy AOrB.fromSlice(s); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + var bc = o.toCell().beginParse().remainingBitsCount(); + return (bc, (o as A|B).toCell().beginParse().remainingBitsCount()); + } + B => { + assert(10>3, 101); + __expect_lazy("[o] load b1 b2 b3"); // loaded because of toCell + return (o.b1 as int, o.toCell().hash() as int); + } + } +} + +@method_id(209) +fun demo209() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] skip (bits16), lazy match ab"); + match (o.ab) { + A => { + return o.ab.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(210) +fun demo210() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] save immutable (tail), skip (bits16), load ab"); // not lazy match because of nested o.toCell() + match (o.ab) { + A => { + return o.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(211) +fun demo211() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] save immutable (tail), load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + assert(o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02030405")), 100); + return o.toCell(); +} + +@method_id(212) +fun demo212() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + __expect_lazy("[o] load initial (bits8) f3, save immutable (tail), load f4"); + if (o.initial > 100 && o.initial < 10000) { + o.f3 = o.f4; + } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02040405")); +} + +@method_id(213) +fun demo213(eval: bool) { + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01080903FFFF")); // more than needed + if (eval && 10>3) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.y += 2; + var s = c.toCell().beginParse(); + return (s.remainingBitsCount(), s.skipBits(8+8).preloadUint(8)); + } else { + throw 123; + } +} + +@method_id(214) +fun demo214() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var s = p.makeCellWrapped().beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(215) +fun demo215() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var s = Point.makeCell(p).beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(216) +fun demo216() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail), load walletId"); + st.seqno += 1; + val w = st.walletId; + st.save(); + return (contract.getData().hash() & 0xFFFF, w); +} + +@method_id(217) +fun demo217() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); // because call mutating method + return p.incXAndMakeCell().beginParse().loadUint(16); +} + +@method_id(218) +fun demo218() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: (1 as int32).toCell(), + f2: 2, + r3: Point{x: 10, y: 20}.toCell(), + f4: 4, + }.toCell()); + __expect_lazy("[o] load (cell?) f2, save immutable (tail), load r3"); + val (f2, pointY) = (o.f2, o.r3.load().y); + o.f2 *= 10; + return (f2, pointY, o.toCell().hash() & 0xFFFF, o.f2); +} + + +@method_id(300) +fun demo300() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail)"); + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0102")); +} + +@method_id(301) +fun demo301() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var x = p.x; + return (x, p.toCell().beginParse().bitsEqual(stringHexToSlice("0102"))); +} + +@method_id(302) +fun demo302() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + p.x = 15; + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0F02")); +} + +@method_id(303) +fun demo303() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail)"); + st.isSignatureAllowed = false; + st.seqno += 1; + return st.toCell().hash(); +} + +@method_id(304) +fun demo304() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail), load walletId publicKey extensions"); + var st2 = st; + (st.isSignatureAllowed, st.seqno) = (false, st.seqno + 1); + return (st2, st.toCell().hash()); +} + +@method_id(305) +fun demo305(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x, save immutable (tail)"); + if (setX != null) { + p.x = setX; + } + return p.toCell().beginParse().loadUint(8); +} + +@method_id(306) +fun demo306(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + if (setX != null) { + p.x = setX; + } + return p; +} + +@method_id(307) +fun demo307() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + return cc.hash(); +} + +@method_id(308) +fun demo308() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + var ll = lazy cc.load(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + __expect_lazy("[ll] lazy match"); + match (ll) { + Point => { + assert(p.x == 1, 100); + p.y += 1; + __expect_lazy("[ll] load x y"); + ll.y += 1; + assert(p.y == ll.y, 101); + ll.y -= 1; + return (p, (ll as PointOrPoint3d).toCell().hash()); + } + Point3d => throw 0 + } +} + +@method_id(309) +fun demo309() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits65) publicKey, save immutable (tail)"); + st.publicKey += 1; + return st.toCell(); +} + +@method_id(310) +fun demo310() { + val cell = NotFixedWidthStorage { + f1: 10, + f2: null, + f3: 20, + f4: true, + f5: createEmptyCell(), + f6: 10, + f7: createAddressNone(), + f8: 9, + f9: "" as bits200, + f10: ("" as bits400, ()), + f11: 0, + }.toCell({skipBitsNValidation: true}); + var st = lazy cell.load(); + var sm = lazy cell.load(); + var sk = lazy cell.load(); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) f6, save immutable (tail), skip (address), load f8"); + match (st.f6) { int8 => st.f6 += 90, bool => throw 123 }; + __expect_lazy("[sm] load f5"); + sm.f5; // previous (data-only) fields are not needed to reach a ref + __expect_lazy("[sk] skip (bits8) (int8?) (bits9) (int8 | bool), load f7"); + sk.f7; // ref (f5) is not needed to be skipped to reach data + return (st.toCell().hash() & 0xFFFFFFFF, st.f8); +} + +@method_id(311) +fun demo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = false; + st.extensions = null; + return st.toCell(); +} + + +@method_id(400) +fun demo400() { + var t = createEmptyTuple(); + t.push(stringHexToSlice("0102")); + t.push(stringHexToSlice("0304")); + t.push(stringHexToSlice("0506")); + var corrX: int = -1; + while (t.size() && corrX == -1) { + var p = lazy Point.fromSlice(t.pop()); + __expect_lazy("[p] load x y"); + if (p.y == 6) { + corrX = p.x; + } + } + if (corrX == -1) { + throw 123; + } + return corrX; +} + +fun helper401(s: slice, fakeHandle: bool): never { + val msg = lazy MsgFullCounter.fromCell(beginCell().storeSlice(s).endCell()); + match (msg) { + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + var newX = 100; + __expect_lazy("[msg] load byValue"); + if (!fakeHandle) { + newX -= msg.byValue; + } else { + newX += msg.byValue; + } + throw newX; + } + CounterIncrement => { + var newX = 50; + if (!fakeHandle) { + __expect_lazy("[msg] load byValue"); + newX += msg.byValue; + } + throw newX; + } + else => throw 0xFFFF + } +} + +@method_id(401) +fun demo401(s: slice, fakeHandle: bool) { + try { + helper401(s, fakeHandle); + } catch (excNo) { + return excNo; + } +} + +@method_id(402) +fun demo402(s: slice) { + try { + val msg = lazy MsgFullCounter.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterDecrementBy1 => {} + CounterReset => {} + CounterDecrement => {} + CounterIncrement => {} + // without else, 0xFFFF is thrown by default + } + return 0; + } catch (excno) { + return excno; + } +} + +@method_id(403) +fun demo403(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + var st = lazy loadWalletStorage(); + var t = createEmptyTuple(); + __expect_lazy("[st] skip (bits1), load seqno walletId"); + match (msg) { + CounterIncrement => t.push(st.seqno), + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + t.push(st.walletId); + if (t.size() == 1) { + __expect_lazy("[msg] load byValue"); + t.push(msg.byValue); + } + } + } + return t; +} + +@method_id(404) +fun demo404(s: slice) { + var msg = lazy MyMsg32.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + var curCounter = 10; + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter32Increment => { + __expect_lazy("[msg] load byValue"); + curCounter += msg.byValue; + } + Counter32DecrementBy1 => { + curCounter -= 1; + } + Counter32Reset => { + __expect_lazy("[msg] load initial, skip (bits8), load f3"); + assert (msg.initial >= 0) throw 0xFF; + if (msg.initial > 100) { + msg.initial = 100; + } + curCounter = msg.initial; + if (msg.f3 == 100) { + curCounter *= 2; + } + } + } + return curCounter; +} + + +@method_id(405) +fun demo405(s: slice) { + var sum = 0; + try { var msg = lazy AutoPrefixedBin2.fromSlice(s); match (msg) { + AutoP3 => { + sum = -1; + } + AutoP1 => { + __expect_lazy("[msg] load f1 f2"); + sum = msg.f1 + msg.f2; + } + AutoP2 => { + __expect_lazy("[msg] load f1 f2 f3"); + sum = msg.f1; + sum += msg.f2; + if (sum > 0) { + sum += msg.f3; + } + } + } } catch { sum = -2 } + return sum; +} + + +@method_id(406) +fun demo406(s: slice) { + try { + match (val msg = lazy ShortPrefixed.fromSlice(s)) { + ShortP1 => { + return 1; + } + ShortP2 => { + __expect_lazy("[msg] skip (bits8), load f2"); + var sum = 2 + msg.f2; + return sum; + } + else => { throw 9999; } + } + } catch(excno) { return excno } +} + +@method_id(407) +fun demo407(s: slice): int { + match (val msg = lazy NotEqPrefixed.fromSlice(s)) { + NotEqPrefixedS1 => return msg.f, + NotEqPrefixedS2 => return msg.f, + NotEqPrefixedS3 => return msg.f, + } + throw 0xFF; +} + +@method_id(408) +fun demo408() { + var o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] skip (bits16), load inner2 wa"); + if (10>3) { + o.inner2.i2 = 10; + } + return o.wa.ab.toCell().beginParse().remainingBitsCount(); +} + +@method_id(409) +fun demo409() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + return (o.initial, f3); +} + +@method_id(410) +fun demo410(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + // via codegen, check that "return" is implicitly added, and IFJMP is generated + match (msg) { + CounterIncrement => { debug.print(1) } + CounterDecrement => { debug.dumpStack() } + CounterReset => { random.setSeed(10) } + CounterDecrementBy1 => { reserveToncoinsOnBalance(10, 1) } + else => { /* do nothing, "just accept tons" */ } + } +} + +@method_id(411) +fun demo411(s: slice) { + val msg = lazy AOrB.fromSlice(s); + // the same, but for either + match (msg) { + A => { debug.print(1) } + B => { debug.dumpStack() } + } +} + + +fun main() { + val s = stringHexToSlice("0102"); + var p = lazy Point.fromSlice(s); + __expect_lazy("[p] skip (bits8), load y"); + return (p.y, s.remainingBitsCount()); // s not changed +} + + +/** +@testcase | 0 | | 2 16 +@testcase | 101 | | 1 +@testcase | 102 | 0 | 2 +@testcase | 103 | | 8 +@testcase | 104 | | 1 4 +@testcase | 105 | x{01041_} | 1 8 +@testcase | 105 | x{01820304} | 1 4 +@testcase | 106 | x{01041_} | -1 (null) +@testcase | 106 | x{01820304} | 0 2 +@testcase | 107 | | 1 2 +@testcase | 108 | | 1 15 1 2 3 4 5 6 7 8 9 +@testcase | 109 | | 0 +@testcase | 110 | x{08} | [ 8 ] +@testcase | 110 | x{88} | [ -8 ] +@testcase | 111 | x{08} 50 | [ 8 ] +@testcase | 111 | x{88} 50 | [ -8 ] +@testcase | 112 | x{0110} | 16 +@testcase | 112 | x{020f} | -15 +@testcase | 112 | x{03} | -1 +@testcase | 112 | x{03ffff} | -1 +@testcase | 112 | x{04000000FF} | 255 +@testcase | 113 | x{0110} | 16 +@testcase | 113 | x{020f} | -15 +@testcase | 113 | x{03} | -1 +@testcase | 113 | x{03ffff} | -1 +@testcase | 113 | x{04000000FF} | 255 +@testcase | 114 | x{0110} | 16 1 +@testcase | 114 | x{020f} | [ -15 ] typeid-27 +@testcase | 114 | x{03} | -1 1 +@testcase | 114 | x{03ffff} | -1 1 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-27 +@testcase | 115 | x{08} | 8 42 +@testcase | 115 | x{88} | 0 2 +@testcase | 115 | x{83} | -1 2 +@testcase | 118 | x{08} | 8 +@testcase | 118 | x{88} | -8 +@testcase | 119 | x{0F} | 15 -1 +@testcase | 119 | x{8F0203} | -1 2 +@testcase | 120 | x{010208} | [ 8 1 ] +@testcase | 120 | x{010288} | [ -8 1 ] +@testcase | 121 | | 22 +@testcase | 122 | x{010208} | 9 1 +@testcase | 122 | x{010288} | -8 1 +@testcase | 123 | x{010280} | 5 +@testcase | 124 | x{010283} | [ -3 2 ] +@testcase | 125 | | 127 32 15 2 3 typeid-1 15 3 -1 +@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-3 32 15 -1 +@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-4 32 -1 2 +@testcase | 127 | | 1 2 -1 255 +@testcase | 128 | x{5500} | -1 -1 +@testcase | 128 | x{558F01024} | 0 2 +@testcase | 129 | | -1 15 +@testcase | 130 | | 456 +@testcase | 131 | | 777 1 +@testcase | 132 | -1 | 889 -1 +@testcase | 132 | 0 | 888 -1 +@testcase | 133 | | 18 +@testcase | 134 | | 15 +@testcase | 135 | -1 | 40 +@testcase | 135 | 0 | 9 +@testcase | 136 | -1 | -1 +@testcase | 136 | 0 | 2 +@testcase | 137 | x{0102} | 1 +@testcase | 137 | x{03} | 3 +@testcase | 137 | x{1} | 109 +@testcase | 138 | x{0102} | 2 +@testcase | 138 | x{03} | 109 +@testcase | 139 | x{0105} 0 | 5 +@testcase | 139 | x{01} 0 | 9 +@testcase | 139 | x{0230} 0 | 48 +@testcase | 139 | x{0230} -1 | 100 +@testcase | 140 | -1 | 4 +@testcase | 140 | 0 | 255 +@testcase | 141 | -1 | 1 +@testcase | 142 | x{010203} 0 | 3 +@testcase | 142 | x{880203} 0 | 8 +@testcase | 142 | x{0102} 0 | 9 +@testcase | 143 | | 17 +@testcase | 144 | 0 | 1 +@testcase | 145 | 0 | 2 +@testcase | 145 | -1 | 5 +@testcase | 146 | | -1 +@testcase | 147 | | -1 0 +@testcase | 148 | | 8 +@testcase | 149 | | 9 +@testcase | 150 | | 1 16 +@testcase | 151 | | -1 +@testcase | 152 | | 32 +@testcase | 153 | | 0 +@testcase | 154 | | [ [ 1 2 ] ] +@testcase | 155 | | 555 +@testcase | 156 | x{0110} | 16 +@testcase | 156 | x{FF10} | -100 +@testcase | 157 | | -1 +@testcase | 158 | | 2 10 255 +@testcase | 159 | | -1 -1 31 -1 31 + +@testcase | 200 | | 1 +@testcase | 201 | | 1 2 -1 +@testcase | 202 | | 127 127 6 +@testcase | 203 | | 1 2 3 4 5 6 7 8 9 +@testcase | 204 | | 10 20 3 40 5 6 7 8 9 +@testcase | 205 | | -1 +@testcase | 206 | | -1 72 +@testcase | 207 | | C{11A18FA6FB377116ABE1827A4469944C29484C8B00B33BDC4E560D405D5BFDAD} +@testcase | 208 | x{0F0203} | 23 24 +@testcase | 209 | | 23 +@testcase | 210 | | 40 +@testcase | 211 | | C{32B95044B7193DA671CFD5822D3CAA23DF5055A7E75622EBCB16D31C23C50AA8} +@testcase | 212 | | -1 +@testcase | 213 | -1 | 48 11 +@testcase | 214 | | 258 0 +@testcase | 215 | | 258 0 +@testcase | 216 | | 12826 456 +@testcase | 217 | | 514 +@testcase | 218 | | 2 20 11636 20 + +@testcase | 300 | | -1 +@testcase | 301 | | 1 -1 +@testcase | 302 | | -1 +@testcase | 303 | | 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 304 | | -1 123 456 0 (null) 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 305 | null | 1 +@testcase | 305 | 88 | 88 +@testcase | 306 | null | 1 2 +@testcase | 306 | 88 | 88 2 +@testcase | 307 | | 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 308 | | 1 3 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 309 | | C{EE7C06B69DDCEFEBED507FE35F603A9019D4AE79A13F670FE4E08928AF484838} +@testcase | 310 | | 3332695883 9 +@testcase | 311 | | C{16A4757FA412AD5E89418F1263904CD57FA4DF0E73EE8A53DC60F862C72BCD9D} + +@testcase | 400 | | 5 +@testcase | 401 | x{0110} -1 | 50 +@testcase | 401 | x{020f} 0 | 85 +@testcase | 402 | x{020f} | 0 +@testcase | 402 | x{ffffffff} | 65535 +@testcase | 403 | x{0110} | [ 123 ] +@testcase | 403 | x{020f} | [ 456 15 ] +@testcase | 404 | x{1234567800000010} | 26 +@testcase | 404 | x{23456789} | 9 +@testcase | 404 | x{345678900000000002030405} | 0 +@testcase | 405 | x{01c220_} | 15 +@testcase | 405 | x{41c20260_} | 24 +@testcase | 405 | x{a0_} | -1 +@testcase | 405 | x{ffff} | -2 +@testcase | 406 | x{7} | 1 +@testcase | 406 | x{40102} | 4 +@testcase | 406 | x{99} | 9999 +@testcase | 407 | x{7F} | 63 +@testcase | 407 | x{1F} | 15 +@testcase | 407 | x{BF} | -1 +@testcase | 408 | | 24 +@testcase | 409 | | 255 3 + +@fif_codegen +""" + demo101() PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 PLDI // p.x + }> +""" + +@fif_codegen +""" + demo103() PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 LDU + NIP // lazyS + 8 PLDI // p.y +""" + +@fif_codegen +""" + demo106() PROC:<{ // s + // lazyS + PUSHNULL // lazyS x + SWAP // x lazyS + 8 LDU + NIP // x lazyS + 1 LDU // x '18 lazyS +""" + +@fif_codegen +""" + demo109() PROC:<{ // + x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS + 65 LDU + NIP // lazyS + 256 PLDU // st.publicKey + }> +""" + +@fif_codegen +""" + demo113() PROC:<{ + // lazyS + x{01} SDBEGINSQ + IF:<{ + 8 PLDI + }>ELSE<{ + x{02} SDBEGINSQ + IF:<{ + 8 PLDI + NEGATE + }>ELSE<{ + x{03} SDBEGINSQ + IF:<{ + DROP + -1 PUSHINT + }>ELSE<{ + x{04} SDBEGINSQ + IFNOTJMP:<{ + 63 THROW + }> + 32 PLDI + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo134() PROC:<{ // + x{010f} PUSHSLICE // lazyS + x{01} SDBEGINSQ // lazyS '6 + 134 THROWIFNOT // lazyS + 8 PLDI // cc + }> +""" + +@fif_codegen +""" + demo136() PROC:<{ + x{0102} PUSHSLICE + SWAP + IFJMP:<{ + DROP + -1 PUSHINT + }> + 8 LDU + NIP + 8 PLDI + }> +""" + +@fif_codegen +""" + demo146() PROC:<{ + x{0000000} PUSHSLICE + PLDOPTREF + ISNULL + }> +""" + +@fif_codegen +""" + demo201() PROC:<{ + x{0102} PUSHSLICE + DUP + 8 LDI + 8 PLDI + s2 PUSH + NEWC + STSLICE + ENDC + s3 PUSH + NEWC + STSLICE + ENDC + s0 s4 XCHG + NEWC + STSLICE + ENDC +""" + + +@fif_codegen +""" + demo203() PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 PLDU // '14 o.f1 + 1 EQINT // '14 '18 + IFJMP:<{ // '14 + NEWC // '14 b + STSLICE // b + ENDC // '21 + CTOS // s +""" + +@fif_codegen +""" + demo206() PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 LDU // '14 o.f1 lazyS + 8 LDU // '14 o.f1 o.f2 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 lazyS + 8 PLDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 +""" + +@fif_codegen +""" + demo215() PROC:<{ + x{0102} PUSHSLICE + 8 LDI + 8 PLDI + SWAP + NEWC + 8 STI + 8 STI + ENDC +""" + +@fif_codegen +""" + demo302() PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + NEWC + x{0f} STSLICECONST + STSLICE +""" + +@fif_codegen +""" + demo311() PROC:<{ // + 777 PUSHINT // '6=777 + contract.getFakeData() CALLDICT // '7 + CTOS // lazyS + 1 LDI // '13 lazyS + NIP // lazyS + 320 PUSHINT // lazyS '14=320 + PLDSLICEX // '15 + PUSHNULL // '15 st.extensions + NEWC + b{0} STSLICECONST // '15 st.extensions b + s1 s2 XCHG // st.extensions '15 b + STSLICE // st.extensions b + STOPTREF // b + ENDC // '21 + }> +""" + +@fif_codegen +""" + demo404() PROC:<{ + // lazyS + x{12345678} SDBEGINSQ + IF:<{ + 32 PLDI + 10 ADDCONST + }>ELSE<{ + x{23456789} SDBEGINSQ + IF:<{ + DROP + 9 PUSHINT + }>ELSE<{ + x{34567890} SDBEGINSQ + IFNOTJMP:<{ + 16 PUSHPOW2DEC + THROWANY + }> + 32 LDU + 8 LDU + NIP + 8 PLDI + OVER + 100 GTINT + IF:<{ + 100 PUSHINT + s2 POP + }> + 100 EQINT + IF:<{ + 1 LSHIFT# + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo410() PROC:<{ + // lazyS + x{01} SDBEGINSQ + IFJMP:<{ + ... + }> + x{02} SDBEGINSQ + IFJMP:<{ + DROP + DUMPSTK + }> + x{04} SDBEGINSQ + IFJMP:<{ + ... + }> + x{03} SDBEGINSQ + NIP + IFJMP:<{ + ... + RAWRESERVE + }> + }> +""" + +@fif_codegen +""" + demo411() PROC:<{ // s + // lazyS + 1 PLDU // '10 + IFJMP:<{ // + DUMPSTK // + }> // + 1 PUSHINT // '11=1 + s0 DUMP DROP // + }> +""" + +@fif_codegen DECLPROC pushToGlobalTup() + */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 700f2a3c5..9b70ae40c 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -1,4 +1,4 @@ -import "imports/use-dicts.tolk" +import "imports/use-dicts.ext.tolk" fun simpleAllConst() { return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); @@ -13,9 +13,9 @@ fun compileTimeEval1(x: int) { @method_id(101) fun withIfNot(x: int, y: int) { - if (!x) { return 10; } - else if (!y) { return 20; } - return x+y; + if (!x) { return 10 } + else if (!y) { { return 20 } } + return x+y } @method_id(102) @@ -120,7 +120,7 @@ fun testLogicalOps2(first: int) { (10 == s.loadUint(32)) || (20 == s.loadUint(32)) || (3 == s.loadUint(32)) || (4 == s.loadUint(32)); sum += s.loadUint(32); } - return (s.getRemainingBitsCount(), sum); + return (s.remainingBitsCount(), sum); } @method_id(112) @@ -148,6 +148,31 @@ fun testConvertIfToIfnot(x: bool) { return -4; } +@pure +fun get123(): int + asm "123 PUSHINT"; + +@method_id(114) +fun test114() { + val fals = (get123() < 0) as int; + return (fals ? -1 : fals) != 0; +} + +@method_id(115) +fun test115() { + val tru = get123() >= 0; + val fals = get123() < 0; + + if ((true || false) && (false || false)) { + throw 123; + } + if ((tru || fals) && (fals || fals)) { + throw 456; + } + return (tru, fals); +} + + fun main() { } @@ -187,11 +212,12 @@ fun main() { @testcase | 112 | 5 0 | 0 -1 0 -1 -1 @testcase | 112 | 0 -1 | -1 -1 -1 -1 -1 @testcase | 113 | 0 | 1 +@testcase | 114 | | 0 +@testcase | 115 | | -1 0 @fif_codegen """ - simpleAllConst PROC:<{ - // + simpleAllConst() PROC:<{ TRUE 0 PUSHINT TRUE @@ -206,8 +232,7 @@ fun main() { @fif_codegen """ - compileTimeEval1 PROC:<{ - // x + compileTimeEval1() PROC:<{ // x DUP // x x 0 EQINT // x '1 FALSE // x '1 '4 @@ -223,9 +248,9 @@ fun main() { @fif_codegen """ - withIfNot PROC:<{ + withIfNot() PROC:<{ // x y c2 SAVE - SAMEALTSAVE // x y + SAMEALTSAVE OVER // x y x IFNOTJMP:<{ // x y 2DROP // @@ -243,8 +268,7 @@ fun main() { @fif_codegen """ - testAndConstCodegen PROC:<{ - // + testAndConstCodegen() PROC:<{ FALSE 0 PUSHINT DUP @@ -266,8 +290,7 @@ fun main() { @fif_codegen """ - testOrConstCodegen PROC:<{ - // + testOrConstCodegen() PROC:<{ -1 PUSHINT TRUE FALSE @@ -292,8 +315,7 @@ For example, `a && b` can be expressed without IFs. These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testAndSimpleCodegen PROC:<{ - // a b + testAndSimpleCodegen() PROC:<{ // a b SWAP // b a IF:<{ // b 0 NEQINT // '2 @@ -306,8 +328,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testOrSimpleCodegen PROC:<{ - // a b + testOrSimpleCodegen() PROC:<{ // a b SWAP // b a 0 GTINT // b '3 IF:<{ // b @@ -322,8 +343,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testConvertIfToIfnot PROC:<{ - // x + testConvertIfToIfnot() PROC:<{ // x DUP // x x 100 THROWIF DUP // x x @@ -331,15 +351,15 @@ These are moments of future optimizations. For now, it's more than enough. DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '5=1 + 1 PUSHINT // '7=1 }> // x DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '6=1 + 1 PUSHINT // '8=1 }> // x 100 THROWIFNOT - -4 PUSHINT // '9=-4 + -4 PUSHINT // '11=-4 }> """ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk new file mode 100644 index 000000000..489cdf748 --- /dev/null +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -0,0 +1,264 @@ +const C1 = 1; +const C2 = C1 + 1; + +@method_id(101) +fun test1(x: int) { + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + } +} + +@method_id(102) +fun test2() { + var x = 3; + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + } +} + +@noinline +fun isGt10(x: int) { return x > 10; } + +@method_id(103) +fun test3(init: int) { + var r1 = match (isGt10(init)) { true => 10, false => 20 }; + var r2 = match (isGt10(init)) { !!true => 10, !!false => 20, else => 8 }; + var r3 = match (isGt10(init)) { 10 > 3 => 10, 10 < 3 => 20, 100 > 1 => 30, else => 8 }; + return (r1, r2, r3); +} + +@method_id(104) +fun test4(x: int?) { + return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 } +} + +@method_id(105) +fun test5(x: int) { + try { + return match (val r = x * 2) { + 10/C1 => 10, + C2*10 => throw 500, + else => 50 + } + } catch (excCode) { + return excCode + } +} + +@method_id(106) +fun test6(x: coins) { + var result1 = 0; + match (x) { + ton("0.05") => { result1 = 5; } + ton("0.1") => { result1 = 10; } + else => { result1 = 20; } + } + var result2 = 0; + match (x) { + ton("0.05") => { result2 = 5 } + ton("0.1") => { result2 = 10 } + } + return (result1, result2); +} + +@method_id(107) +fun test7(x: int8) { + match (x) { 10 => {} } + return match (x) { + 10 => 10, + 20 as int8 => 20, + 30 as int16 => 30, + 40 as uint255 => 40, + else => 50, + }; +} + +@method_id(108) +fun test8(x: int) { + var r1 = match (x) { else => 10 }; + match (x) { } + match (x) { else => { r1 += 20; } } + return r1; +} + +@method_id(109) +fun test9(x: int | slice) { + if (x is slice) { + match (x.loadInt(32)) { -1 => {}, 0 => {} } + } + return match (x) { + int => match (x) { 10 => 5, else => -1 }, + slice => match (x.loadInt(32)) { -1 => 0, 0 => 0, else => throw 20 } + } +} + +@method_id(110) +fun test10(x: int) { + var result1 = match (x) { + 10 => throw 10, + 20 => return 20, + 30 => { x += 10; return x; } + else => x + }; + return result1 * 2; +} + +@method_id(111) +fun test11(x: bool) { + return match (x) { + true => false, + else => !x, + } +} + +global g12: int; + +@noinline +fun helper12(x: int) { + // via codegen, check that `return` is added implicitly to every case, and `IFJMP` is produced + match (x) { + 1 => g12 = 1, + 2 => g12 = 2, + 3 => g12 = 3, + else => g12 = x, + } +} + +@method_id(112) +fun test12(x: int) { + g12 = 0; + helper12(x); + return g12 +} + + +type asdf = int; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + return match (10) { + asdf => 2, // also match by type, regardless of local var + }; +} + +/** +@testcase | 0 | | 2 +@testcase | 101 | 1 | 100 +@testcase | 101 | 2 | 200 +@testcase | 101 | 3 | 300 +@testcase | 101 | 4 | 400 +@testcase | 102 | | 300 +@testcase | 103 | 5 | 20 20 20 +@testcase | 103 | 15 | 10 10 10 +@testcase | 104 | 15 | 30 +@testcase | 104 | null | 0 +@testcase | 105 | 10 | 500 +@testcase | 105 | 11 | 50 +@testcase | 106 | 50000000 | 5 5 +@testcase | 106 | 300 | 20 0 +@testcase | 107 | 30 | 30 +@testcase | 107 | 40 | 40 +@testcase | 107 | 41 | 50 +@testcase | 108 | 5 | 30 +@testcase | 109 | 1 1 | -1 +@testcase | 110 | 20 | 20 +@testcase | 110 | 30 | 40 +@testcase | 110 | 50 | 100 +@testcase | 111 | 0 | -1 +@testcase | 111 | -1 | 0 +@testcase | 112 | 2 | 2 +@testcase | 112 | 20 | 20 + +@fif_codegen +""" + test1() PROC:<{ // x + DUP // x x + 1 EQINT // x '3 + IF:<{ // x + DROP // + 100 PUSHINT // '1=100 + }>ELSE<{ // x + DUP // x x + 2 EQINT // x '8 + IF:<{ // x + DROP // + 200 PUSHINT // '1=200 + }>ELSE<{ // x + 3 EQINT // '13 + IF:<{ // + 300 PUSHINT // '1=300 + }>ELSE<{ // + 400 PUSHINT // '1=400 + }> + }> + }> + }> +""" + +@fif_codegen +""" + test2() PROC:<{ + 300 PUSHINT + }> +""" + +@fif_codegen +""" + test3() PROC:<{ // init + DUP // init init + isGt10() CALLDICT // init '2 + -1 EQINT // init '5 + IF:<{ // init + 10 PUSHINT // init '3=10 + }>ELSE<{ // init + 20 PUSHINT // init '3=20 + }> // init r1 +""" + +@fif_codegen +""" + test8() PROC:<{ // x + DROP // + 30 PUSHINT // r1 + }> +""" + +@fif_codegen +""" + helper12() PROC:<{ + DUP + 1 EQINT + IFJMP:<{ + DROP + 1 PUSHINT + $g12 SETGLOB + }> + DUP + 2 EQINT + IFJMP:<{ + DROP + 2 PUSHINT + $g12 SETGLOB + }> + DUP + 3 EQINT + IFJMP:<{ + DROP + 3 PUSHINT + $g12 SETGLOB + }> + $g12 SETGLOB + }> +""" + + */ diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk new file mode 100644 index 000000000..45278ad64 --- /dev/null +++ b/tolk-tester/tests/meaningful-1.tolk @@ -0,0 +1,49 @@ +struct CounterReset { + initialValue: int64; +} + +struct CounterIncrement1 { +} + +struct CounterIncrement { + byValue: int32; +} + +type MyMessage = + CounterIncrement + | CounterIncrement1 + | CounterReset; + +fun fakeParseMessage(mode: int): MyMessage { + if (mode == 1) { + return CounterIncrement1 {}; + } else if (mode > 0) { + return CounterIncrement { byValue: mode }; + } else { + return CounterReset { initialValue: 0 }; + } +} + +fun onInternalMessage(curCounter: int, mode: int) { + var m = fakeParseMessage(mode); + __expect_type(m, "MyMessage"); + + val nextCounter: int = match (m) { + CounterReset => m.initialValue, + CounterIncrement => curCounter + m.byValue, + CounterIncrement1 => curCounter + 1, + }; + return nextCounter; +} + +/** +@testcase | 0 | 50 0 | 0 +@testcase | 0 | 50 1 | 51 +@testcase | 0 | 50 9 | 59 + +@fif_codegen +""" + onInternalMessage() PROC:<{ // curCounter mode + fakeParseMessage() CALLDICT // curCounter m.USlot1 m.UTag +""" + */ diff --git a/tolk-tester/tests/method_id.tolk b/tolk-tester/tests/method_id.tolk deleted file mode 100644 index e7e70d245..000000000 --- a/tolk-tester/tests/method_id.tolk +++ /dev/null @@ -1,17 +0,0 @@ -@method_id(1) -fun foo1(): int { return 111; } -@method_id(3) -fun foo2(): int { return 222; } -@method_id(10) -fun foo3(): int { return 333; } -@method_id(11) -fun slice(slice: slice): slice { return slice; } -fun main(): int { return 999; } - -/** - method_id | in | out -@testcase | 1 | | 111 -@testcase | 3 | | 222 -@testcase | 10 | | 333 -@testcase | 0 | | 999 -*/ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk new file mode 100644 index 000000000..4b09a53d6 --- /dev/null +++ b/tolk-tester/tests/methods-tests.tolk @@ -0,0 +1,315 @@ +import "@stdlib/common.tolk" +type MInt = int; + +global callOrder: tuple; + +fun int.zero() { return 0; } +fun int.plus1(self) { return self + 1; } +fun MInt.plusN(self, n: int) { return self + n; } +fun int.main(self) { return self; } + +struct Point { + x: int; + y: int; +} + +fun Point.create0() { return Point.create(0, 0); } +fun Point.create(x: int, y: int): Point { return {x,y}; } + +fun Point.getMaxCoord(self) { return self.x > self.y ? self.x : self.y; } +fun Point.incX(mutate self) { self.x += 1; } +fun Point.incXMutateY(mutate self, mutate curY: int) { self.x += 1; curY = self.y; } + +fun Point?.isNull(self) { return self == null; } + +struct Pair { + first: A; + second: B; +} + +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +@noinline +fun Pair.createFrom(first: U, second: V): Pair { + return { first: first as A, second: second as B }; +} +fun Pair.is0(self) { + return self.first == 0 && self.second == 0; +} +@noinline +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +type Tuple2Int = [int, int]; +fun [int, int].getLast(self) { return self.1; } +fun [int, MInt].assignLast(mutate self, v: int) { self.1 = v; } +fun Tuple2Int.getSum(self) { return self.0 + self.1; } + +fun int.calcExactF(self) { return self; } +fun int8.calcExactF(self) { return self * 2; } +fun int8?.calcExactF(self) { return self == null ? 0 : self * 3; } + +fun T.copy(self) { return self; } +fun T.assign(mutate self, value: T2) { self = value; } + +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } +fun Wrapper.getItem(self) { return self.item; } +fun Wrapper.getItem(self) { return 100500; } +fun createWrapper(item: T) { return Wrapper.create(item); } + +@method_id(101) +fun test1() { + var i = int.zero() + MInt.zero(); + i = i.plus1(); + return (i.plus1(), i, i.plusN(5), i.plusN(i.plus1())); +} + +@method_id(102) +fun test2(a: int8, b: int16) { + var c = b as coins?; + return (a.plus1(), b.plusN(a), c!.plus1()); +} + +@method_id(103) +fun test3() { + var p = Point.create0(); + var yy = 10; + p.incX(); + Point.incX(mutate p); + p.incXMutateY(mutate yy); + p.y = p.x; + Point.incXMutateY(mutate p, mutate yy); + return (p, p.getMaxCoord(), Point.getMaxCoord(p), yy); +} + +@method_id(104) +fun test4() { + var p0 = Point.create0(); + var p5 = Point.create(5,5) as Point?; + var pN = null as Point?; + return (p0.isNull(), p5.isNull(), pN.isNull(), Point.isNull(null), Point.isNull({x:3,y:3})); +} + +@method_id(105) +fun test5(a: MInt, b: int) { + var t = [a,b]; + return (t.getSum(), t.assignLast(a+b), t.getSum(), Tuple2Int.getLast(t)); +} + +@method_id(106) +fun test6() { + var x = 5; + var x8 = x as int8; + return ( + x.calcExactF(), x8.calcExactF(), (8 as int8?).calcExactF(), (null as int8?).calcExactF(), + (5 as int16).calcExactF(), (x8 as coins).calcExactF(), (x8 as MInt).calcExactF(), + int8.calcExactF(x) + ); +} + +@method_id(107) +fun test7() { + __expect_type(5.copy(), "int"); + __expect_type((5 as int99).copy(), "int99"); + __expect_type(beginCell().storeInt(1,32).copy(), "builder"); + __expect_type(((null, null) as (MInt?,bool?)).copy(), "(MInt?, bool?)"); + return ( + beginCell().storeInt(1,32).endCell().beginParse().loadInt(32).copy() + 1, + stringHexToSlice("01").copy().loadInt(8), + stringHexToSlice("FF").loadInt(8), + ); +} + +@method_id(108) +fun test8(x: int32) { + var cp = x.copy(); + x.assign(10); + int32.assign(mutate x, x+cp); + (x as int18).assign(x+2); + __expect_type(x, "int32"); + return (x, cp); +} + +@method_id(109) +fun test9() { + var w1 = Wrapper.create(5); + var w2 = Wrapper.create(10); + var w3 = createWrapper(20 as int16); + var w4 = Wrapper>.create({item:beginCell().storeInt(30,32).storeInt(31,32).endCell().beginParse()}); + __expect_type(w1, "Wrapper"); + __expect_type(w2, "Wrapper"); + __expect_type(w3, "Wrapper"); + __expect_type(w4, "Wrapper>"); + __expect_type(w4.getItem(), "Wrapper"); + __expect_type(w4.getItem().getItem(), "slice"); + return ( + w1.getItem(), w2.getItem(), w3.getItem(), w4.item.item.loadInt(32), + Wrapper.getItem(w1), w1.getItem(), Wrapper>.getItem(w4).getItem().loadInt(32) + ); +} + + +fun T.isAnotherNull(self) { callOrder.push(1); return self == null; } +fun int.isAnotherNull(self) { callOrder.push(2); return false; } +fun int?.isAnotherNull(self) { callOrder.push(3); return false; } + +@method_id(110) +fun test10() { + callOrder = createEmptyTuple(); + ( + 5.isAnotherNull(), (5 as int?).isAnotherNull(), beginCell().isAnotherNull(), (beginCell() as builder?).isAnotherNull(), + null.isAnotherNull(), true.isAnotherNull(), ((1,null) as (int,slice?)).isAnotherNull() + ); + return (callOrder, null.isNull()); +} + +fun int.anyMethod(self) { callOrder.push(1); } +fun slice?.anyMethod(self) { callOrder.push(2); } +fun T.anyMethod(self) { callOrder.push(3); } +fun callAnyMethod(obj: T) { return obj.anyMethod(); } + +@method_id(111) +fun test11() { + callOrder = createEmptyTuple(); + (callAnyMethod(10), callAnyMethod(null as slice?), callAnyMethod(null), + callAnyMethod(null), callAnyMethod(null)); + return callOrder; +} + +fun int?.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return beginCell(); } +fun Wrapper>.arbitraryMethod(self) { return beginCell(); } +fun T?.arbitraryMethod(self): never { throw 123; } + +fun test12() { + __expect_type(10.arbitraryMethod(), "int?"); + __expect_type((10 as int8?).arbitraryMethod(), "int?"); + __expect_type((10 as coins?)!.arbitraryMethod(), "int?"); + + __expect_type(null.arbitraryMethod(), "int?"); + __expect_type((10 as coins?).arbitraryMethod(), "int?"); + + __expect_type(Wrapper { item: 10 }.arbitraryMethod(), "Wrapper"); + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "Wrapper"); + + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "builder"); + __expect_type(Wrapper { item: Wrapper { item: 10 } }.arbitraryMethod(), "builder"); + + __expect_type((Wrapper { item: 10 } as Wrapper?).arbitraryMethod(), "never"); + __expect_type((null as slice?).arbitraryMethod(), "never"); +} + +type BuilderOrInt = |builder|int +fun|int|builder.isInt(self) { return self is int; } + +@method_id(113) +fun test13() { + var u1 = beginCell() as|builder|int; + var u2 = 10 as int|builder; + var u3 = 20 as BuilderOrInt; + return (u1.isInt(), u2.isInt(), u3.isInt(), 10.isInt(), beginCell().isInt(), (8 as int8).isInt()); +} + +fun (int|builder|T).isBuilder(self) { return self is builder; } + +@method_id(114) +fun test14() { + var b1 = beginCell() as BuilderOrInt | slice; + var b2 = beginCell() as builder|slice|int|Point; + var i1 = 5 as |int|builder|continuation|(int, int); + var c = beginCell().endCell() as cell?|builder|int; + var cn = null as cell?|int|int8|builder; + return (b1.isBuilder(), b2.isBuilder(), i1.isBuilder(), c.isBuilder(), cn.isBuilder()); +} + +@method_id(115) +fun test15() { + var p1 = Pair.create0(); + __expect_type(p1, "Pair"); + var p2 = Pair.create0(); + __expect_type(p2, "Pair"); + var p3 = Pair.createFrom(10, (20 as coins?)!); + __expect_type(p3, "Pair"); + return (p2.is0(), p2.compareWith(p3.first, p3.second)); +} + +struct CounterIncrement { inc_by: int; } +struct CounterReset { initial_value: int; } +type CounterMsg = CounterIncrement | CounterReset; + +@noinline +fun CounterReset.onInternalMessage(self) { + throw 123; +} +@noinline +fun CounterMsg.onInternalMessage(self) { + return match (self) { + CounterIncrement => self.inc_by, + CounterReset => self.initial_value, + }; +} + +@method_id(116) +fun test16() { + var inc: CounterIncrement = { inc_by: 100 }; + var msg = inc as CounterMsg; + return (inc.onInternalMessage(), msg.onInternalMessage(), CounterMsg.onInternalMessage(CounterReset{initial_value:5})); +} + +fun int.`0`() { return 0; } +fun int.`1`(self) { return self + 1; } + +@method_id(117) +fun test17() { + var zero = int.0(); + return zero.1() + int.0().1() + int.1(int.0()); +} + +fun Wrapper.createNull(): Wrapper { return { item: null }; } +fun Wrapper.createFrom(item: U): Wrapper { return {item}; } + +@method_id(118) +fun test18() { + __expect_type(10.copy, "(int) -> int"); + __expect_type(Wrapper{item:null as int8?}.copy, "(Wrapper) -> Wrapper"); + __expect_type(Wrapper.createNull, "() -> Wrapper"); + __expect_type(Wrapper?>.createNull, "() -> Wrapper?>"); + __expect_type(Wrapper.createFrom, "(int8) -> Wrapper"); + + var cb = Wrapper.createFrom; + return cb(10); +} + +fun main() {} + +/** +@testcase | 101 | | 2 1 6 3 +@testcase | 102 | 5 6 | 6 11 7 +@testcase | 103 | | 4 3 4 4 3 +@testcase | 104 | | 0 0 -1 -1 0 +@testcase | 105 | 5 6 | 11 16 11 +@testcase | 106 | | 5 10 24 0 5 5 5 10 +@testcase | 107 | | 2 1 -1 +@testcase | 108 | 5 | 17 5 +@testcase | 109 | | 5 100500 20 30 5 5 31 +@testcase | 110 | | [ 2 3 1 1 3 1 1 ] -1 +@testcase | 111 | | [ 1 2 3 3 2 ] +@testcase | 113 | | 0 -1 -1 -1 0 -1 +@testcase | 114 | | -1 -1 0 0 0 +@testcase | 115 | | -1 0 +@testcase | 116 | | 100 100 5 +@testcase | 117 | | 3 +@testcase | 118 | | 10 + +@fif_codegen DECLPROC Pair.createFrom() +@fif_codegen DECLPROC Pair.compareWith() +@fif_codegen DECLPROC CounterMsg.onInternalMessage() + */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 9ebf8b1d4..f3e93b694 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -1,8 +1,22 @@ -fun incrementInPlace(mutate self: int, byValue: int): void { +@noinline +fun incrementInPlace(mutate x: int, byValue: int): void { + x = x + byValue; +} + +@noinline +fun int.incrementInPlace(mutate self, byValue: int): void { self = self + byValue; } -fun incrementTwoInPlace(mutate self: int, mutate y: int, byValue: int): int { +@noinline +fun incrementTwoInPlace(mutate x: int, mutate y: int, byValue: int): int { + x.incrementInPlace(byValue); + y += byValue; + return x + y; +} + +@noinline +fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { self.incrementInPlace(byValue); y += byValue; return self + y; @@ -30,13 +44,14 @@ fun testIncrement2() { } +@noinline fun load_next(mutate cs: slice): int { - return loadInt(mutate cs, 32); + return cs.loadInt(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; @inline_ref @@ -58,7 +73,7 @@ fun testSlices1() { return (first, cs.myLoadInt(32), cs.loadInt(32)); } -fun load_decimal_symbol(mutate self: slice): int { +fun slice.load_decimal_symbol(mutate self): int { // load decimal from bits using utf-8 table var n: int = self.loadUint(8); n = n - 48; @@ -113,13 +128,18 @@ fun testNameShadowing() { return (x, sum); } -fun updateTwoItems(mutate self: (int, int), byValue: int) { +type Pair2 = (int, int); +fun Pair2.updateTwoItems(mutate self, byValue: int) { val (first, second) = self; self = (first + byValue, second + byValue); } +fun updateTwoItems(mutate pair: Pair2, byValue: int) { + val (first, second) = pair; + pair = (first + byValue, second + byValue); +} -global t107_1: int; -global t107_2: int; +global t107_1: int +global t107_2: int @method_id(107) fun testMutableTensor() { @@ -134,18 +154,29 @@ fun testMutableTensor() { } @pure -fun myStoreUint(mutate self: builder, x: int, len: int): self +fun builder.myStoreUint(mutate self, x: int, len: int): self asm(x self len) "STIX"; @pure -fun myStoreU32(mutate self: builder, x: int): self { +fun myStoreUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; +@pure +fun storeUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; + +@pure +fun builder.myStoreU32(mutate self, x: int): self { return self.storeUint(x, 32); } +@pure +fun myStoreU32(mutate b: builder, x: int) { + b.storeUint(x, 32); +} fun getSumOfNumbersInCell(c: cell): int { var sum = 0; var s = c.beginParse(); - var n_numbers = s.getRemainingBitsCount() / 32; + var n_numbers = s.remainingBitsCount() / 32; repeat (n_numbers) { sum += s.loadUint(32); } @@ -161,7 +192,7 @@ fun testStoreChaining() { b = b.storeUint(8, 32); b = b.storeUint(9, 32).storeUint(10, 32); - return getBuilderBitsCount(b); + return b.bitsCount(); } @method_id(111) @@ -185,7 +216,7 @@ fun testStoreChainingCustom() { return (sum1, sum2); } -fun myStoreU32_and_mutate_x(mutate self: builder, mutate x: int): void { +fun builder.myStoreU32_and_mutate_x(mutate self, mutate x: int): void { return myStoreUint(mutate self, x += 10, 32); } @@ -215,7 +246,7 @@ fun testStoreChainingForGlobal() { ccc = ccc.myStoreU32(8); ccc = ccc.storeUint(9, 32).myStoreUint(10, 32); - return getBuilderBitsCount(ccc); + return ccc.bitsCount(); } fun alwaysThrows(): int { throw 123; return 123; } @@ -233,10 +264,10 @@ fun testLoadIntForTemporaryObject() { } @pure -fun myStoreUint_pure(mutate self: builder): void +fun builder.myStoreUint_pure(mutate self): void asm "STIX"; -fun myStoreUint_impure(mutate self: builder): void +fun builder.myStoreUint_impure(mutate self): void asm "STIX"; fun testStoreUintPureUnusedResult() { @@ -257,11 +288,11 @@ fun testStoreUintImpureUnusedResult() { global counter: int; -fun writeNext2(mutate self: builder): self { +fun builder.writeNext2(mutate self): self { return self.storeUint(counter += 1, 32).storeUint(counter += 1, 32); } -fun resetCounter(mutate self: builder): self { +fun builder.resetCounter(mutate self): self { counter = 0; return self; } @@ -270,7 +301,7 @@ fun resetCounter(mutate self: builder): self { fun testExplicitReturn() { counter = 0; return ( - beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell().getSumOfNumbersInCell(), + getSumOfNumbersInCell(beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell()), counter ); } @@ -295,28 +326,26 @@ fun main(){} @fif_codegen """ - incrementInPlace PROC:<{ - // self byValue + int.incrementInPlace() PROC:<{ // self byValue ADD // self }> """ @fif_codegen """ - testIncrement2 PROC:<{ + testIncrement2() PROC:<{ ... - incrementTwoInPlace CALLDICT // x y sum1 + incrementTwoInPlace() CALLDICT // x y sum1 -ROT 10 PUSHINT // sum1 x y '11=10 - incrementTwoInPlace CALLDICT // sum1 x y sum2 + int.incrementTwoInPlace() CALLDICT // sum1 x y sum2 s1 s3 s0 XCHG3 // x y sum1 sum2 }> """ @fif_codegen """ - load_next PROC:<{ - // cs + load_next() PROC:<{ // cs 32 LDI // '4 cs SWAP // cs '4 }> @@ -324,16 +353,14 @@ fun main(){} @fif_codegen """ - testStoreUintPureUnusedResult PROC:<{ - // + testStoreUintPureUnusedResult() PROC:<{ 0 PUSHINT // '11=0 }> """ @fif_codegen """ - testStoreUintImpureUnusedResult PROC:<{ - // + testStoreUintImpureUnusedResult() PROC:<{ NEWC // b STIX // '2 DROP // diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 409bb342c..3a703dc7c 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -1,32 +1,34 @@ -const int10:int=10; +const int10:int=10 const dd=1 fun just10(): int { return int10; } -fun eq(v: int): int { return`v`; } +fun eq(v: int): int { return`v` } -@method_id(101) fun `get_-1` (): int {return-1;} -@method_id(102) fun `get_--1` (): int {return--1;} -@method_id(103) fun `get_---1`(): int {return---1;} -@method_id(104) fun `get_+++1`(): int {return+++1;} -@method_id(105) fun `get_+-+1`(): int {return+-+1;} +const ONE = 1struct Point{x:int y:int} + +@method_id(101,) fun `get_-1` (): int {return-1} +@method_id(102,) fun `get_--1` (): int {return- -1} +@method_id(103,) fun `get_---1`(): int {return-(-(-1))} +@method_id(104,) fun `get_+++1`(): int {return+(+(+1))} +@method_id(105,) fun `get_+-+1`(): int {return+-+1} global `some()var`:int; @method_id(110) fun `some_math`(): int { - `some()var`=--6; - return 1*-2*-3*-4*just10()*-5+-`some()var`+--`some()var`---`some()var`; + `some()var`=- -6; + return ONE*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); } @method_id(111) fun `negative_nums`(a:int):int { var m$0:int=1; var m1:int=-(+0x1)*m$0; - return `a`*-1*-(1)*---(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)----0x1; + return `a`*-1*-(1)*-(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)-(-0x1); } @method_id(112) fun `bitwise~ops`(flags:int):[bool,bool] { return[ - (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), + (just10()-3==just10()-(4)- -1)|((2==2)&(eq(eq(10)) -3==just10()-(-13))), ((flags&0xFF)!=0) - ]; + ] } @method_id(113)fun`unary+bitwise-constant`():[int,int,int]{ @@ -37,6 +39,7 @@ global `some()var`:int; return [~-~~+-c3, ~+c3-~`c9`, -(-~+-c20-~c10+c3+~c38&39)]; } +@noinline fun add3(a: int, b: int, c: int) { return a+b+c; } @method_id(115) fun unary_const_check(): [int,int] { @@ -47,8 +50,8 @@ fun add3(a: int, b: int, c: int) { return a+b+c; } return [add3(fst2,snd2,trd2),add3(fst1,snd1,trd1)]; } -fun `load:u32`(mutate self: slice): int { - return self.loadUint(32); +fun slice.`load:u32`(mutate self): int { + return self.loadUint(32) } @method_id(116) fun `call_~_via_backticks`():[int,int,int,int] { @@ -79,16 +82,14 @@ fun`main`(){} @fif_codegen """ - get_+-+1 PROC:<{ - // + get_+-+1() PROC:<{ -1 PUSHINT }> """ @fif_codegen """ - unary+bitwise-constant PROC:<{ - // + unary+bitwise-constant() PROC:<{ -4 PUSHINT 6 PUSHINT -4 PUSHINT @@ -98,15 +99,14 @@ fun`main`(){} @fif_codegen """ - unary_const_check PROC:<{ - // + unary_const_check() PROC:<{ -1 PUSHINT // fst1=-1 DUP // fst1=-1 snd1=-1 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 s1 s1 s0 PUSH3 // fst1=-1 snd1=-1 trd1=2 fst2=-1 snd2=-1 trd2=2 - add3 CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 + add3() CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 3 -ROLL // '13 fst1=-1 snd1=-1 trd1=2 - add3 CALLDICT // '13 '14 + add3() CALLDICT // '13 '14 PAIR // '12 }> """ diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index 65890a92c..c21da1d0e 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -14,8 +14,8 @@ fun test1() { (_, _) = (null, null); var t = createEmptyTuple(); do { - var num: int = numbers.listNext(); - t.tuplePush(num); + var (num: int, numbers redef) = listSplit(numbers!); + t.push(num); } while (numbers != null); return (h, numbers == null, t); @@ -89,10 +89,10 @@ fun main() { @testcase | 103 | 15 | -1 @testcase | 104 | | (null) @testcase | 107 | | -11 + @fif_codegen """ - test1 PROC:<{ - // + test1() PROC:<{ PUSHNULL // numbers 1 PUSHINT // numbers '2=1 SWAP // '2=1 numbers @@ -113,18 +113,16 @@ fun main() { @fif_codegen """ - main PROC:<{ - // + main() PROC:<{ 1 PUSHINT // '3=1 }> """ @fif_codegen """ - test7 PROC:<{ + test7() PROC:<{ ... - LDOPTREF // b '9 '8 - DROP // b c + PLDOPTREF // b c ISNULL // b '11 10 MULCONST // b '13 SWAP // '13 b diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index d0720273a..5b0394542 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -1,3 +1,7 @@ +type TensorOf2IntNull = (int?, int?); +type MInt = int; +type MIntN = int?; + fun getNullableInt(): int? { return 5; } fun sumOfNullableTensorComponents(t: (int, int)?): int { @@ -5,11 +9,25 @@ fun sumOfNullableTensorComponents(t: (int, int)?): int { return t!.0 + t!.1; } +@noinline fun isTensorNull(t: (int, int)?) { return t == null; } -fun incrementNullableTensorComponents(mutate self: (int, int)?): self { +@noinline +fun (int, int)?.isTensorNull(self) { + return self == null; +} + +@noinline +fun incrementNullableTensorComponents(mutate t: (int, int)?) { + if (t != null) { + t!.0 += 1; + t!.1 += 1; + } +} + +fun (int, int)?.incrementNullableTensorComponents(mutate self): self { if (self != null) { self!.0 += 1; self!.1 += 1; @@ -17,12 +35,19 @@ fun incrementNullableTensorComponents(mutate self: (int, int)?): self { return self; } -fun incrementTensorComponents(mutate self: (int, int)): self { +fun incrementTensorComponents(mutate t: (int, int)) { + t.0 += 1; + t.1 += 1; +} + +@noinline +fun (int, int).incrementTensorComponents(mutate self): self { self.0 += 1; self.1 += 1; return self; } +@noinline fun assignFirstComponent(mutate t: (int, int), first: int) { t!.0 = first; } @@ -116,6 +141,10 @@ fun isTensorNullGen(t: (T1, T2)?) { return t == null; } +fun (T1, T2)?.isTensorNullGen(self) { + return self == null; +} + @method_id(109) fun test109() { var x1 = (1, 2); @@ -124,7 +153,7 @@ fun test109() { return ( isTensorNullGen(x1), isTensorNullGen(x2), isTensorNullGen(null), isTensorNullGen(x1), isTensorNullGen(x3), - x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), null.isTensorNullGen() + x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), (null as (int,int)?).isTensorNullGen() ); } @@ -385,7 +414,9 @@ fun getNormalNullableTensorWidth1(vLess100: int?): ([int?], ())? { return ([vLess100], ()); // such a nullable tensor can store NULL in the same slot } -fun getTrickyNullableTensorWidth1(vLess100: int?): (int?, ())? { +type TrickyActually2SlotTensor = (MIntN, ())?; + +fun getTrickyNullableTensorWidth1(vLess100: int?): TrickyActually2SlotTensor { if (vLess100 != null && vLess100 >= 100) { return null; } @@ -415,78 +446,131 @@ fun test135() { t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null); } +@method_id(136, ) +fun test136(x: int?, ) { + var t1 = (x, ); // it's not a tensor with 1 element + __expect_type(t1, "int?", ); + return ((t1, t1 == null, ), ); // testing trailing comma everywhere :) +} +@method_id(140) +fun test140(x: int8, y: int16) { + var t1: (int32, int64)? = ((x, y) as (int32, int64)) as (int32, int64)?; + var t2: (int32, int64)? = null as (int32, int64)?; + return (t1, t2); +} + +@method_id(141) +fun test141() { + var t: TensorOf2IntNull = (1, 2); + if (t.0 != null) { + t.0 = null; + } + t.1 != null ? t.1 = 10 : t.1 = null; + return t; +} + +type T142 = (int, MInt)?; + +@method_id(142) +fun test142() { + var t = (1, 2) as T142; + __expect_type(t!, "(int, MInt)"); + var sum1 = sumOfNullableTensorComponents(t); + var sum2 = sumOfTensor(t!); + return (sum1, sum2, t!); +} + +@method_id(143) +fun test143(setBNull: bool, setANullMid: bool) { + var a: (int, int | slice | null, int)? = (1, 2, 3); + var b = setBNull ? null : a; + if (setANullMid) { + a.1 = null; + } + if (a.1 != null && b != null) { + b.0 += match (a.1) { int => 100, slice => 200 }; + } + __expect_type(a, "(int, int | slice | null, int)"); + __expect_type(b, "(int, int | slice | null, int)?"); + return (a, 777, b); +} fun main(){} /** -@testcase | 101 | | 1 2 -1 -@testcase | 102 | | 1 2 -1 (null) (null) 0 +@testcase | 101 | | 1 2 typeid-1 +@testcase | 102 | | 1 2 typeid-1 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 -1 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-3 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 -@testcase | 108 | 5 6 | 7 8 10 11 -1 (null) (null) 0 +@testcase | 108 | 5 6 | 7 8 10 11 typeid-1 (null) (null) 0 @testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 -@testcase | 110 | | 3 4 (null) (null) 0 6 7 -1 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 typeid-1 @testcase | 111 | | 50 30 70 90 100 @testcase | 112 | | 12 22 @testcase | 113 | | -1 @testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 -@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 -1 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 typeid-1 0 @testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 -@testcase | 116 | 0 | 1 2 -1 1 2 -1 +@testcase | 116 | 0 | 1 2 typeid-1 1 2 typeid-1 @testcase | 117 | | (null) 1 3 -@testcase | 118 | 5 | 5 10 -1 +@testcase | 118 | 5 | 5 10 typeid-1 @testcase | 118 | null | (null) (null) 0 -@testcase | 119 | | (null) (null) 1 2 -1 100 +@testcase | 119 | | (null) (null) 1 2 typeid-1 100 @testcase | 120 | -1 | (null) (null) 0 -@testcase | 120 | 0 | 1 2 -1 +@testcase | 120 | 0 | 1 2 typeid-1 @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 -1 -@testcase | 124 | 0 | 1 3 4 -1 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 typeid-4 +@testcase | 124 | 0 | 1 3 4 typeid-4 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 -@testcase | 127 | 2 | 1 2 3 -1 4 +@testcase | 127 | 2 | 1 2 3 typeid-1 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 -1 +@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-11 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 -1 4 -1 +@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-11 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 -@testcase | 130 | 0 | 1 2 3 -1 +@testcase | 130 | 0 | 1 2 3 typeid-1 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | -1 777 0 777 777 777 0 0 -1 -1 777 -1 -1 -1 777 +@testcase | 131 | | typeid-12 777 0 777 777 777 0 0 typeid-12 typeid-12 777 typeid-12 typeid-12 typeid-13 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 -@testcase | 134 | | 11 21 -1 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 134 | | 11 21 typeid-1 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-15 (null) typeid-15 (null) 0 777 10 typeid-16 (null) typeid-16 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 136 | 9 | 9 0 +@testcase | 136 | null | (null) -1 +@testcase | 140 | 8 9 | 8 9 typeid-17 (null) (null) 0 +@testcase | 141 | | (null) 10 +@testcase | 142 | | 3 3 1 2 +@testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-18 @fif_codegen """ - isTensorNull PROC:<{ - // t.0 t.1 t.NNFlag - 2 1 BLKDROP2 // t.NNFlag + isTensorNull() PROC:<{// t.0 t.1 t.UTag + 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> """ @fif_codegen """ - test113 PROC:<{ - // + test113() PROC:<{ 1 PUSHINT // '2=1 PUSHNULL // '2=1 '3 PAIR // t 1 INDEX // '5 PUSHNULL // '5 '6 0 PUSHINT // '5 '6 '7=0 - isTensorNull CALLDICT // '8 + isTensorNull() CALLDICT // '8 }> """ */ diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index 24aa7f8ae..39906eae4 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -1,13 +1,21 @@ +type MIntN = int?; -fun getNullable4(): int? { return 4; } +fun getNullable4(): MIntN { return 4; } fun getNullableIntNull(): int? asm "PUSHNULL"; -fun eqInt(x: int) { return x; } -fun eq(x: T) { return x; } +fun eqInt(x: int) { return x } +fun eq(x: T) { return x } fun unwrap(x: T?): T { return x!; } fun intOr0(x: int?): int { return null == x ? 0 : x!; } +@pure +fun storeInt(mutate b: builder, x: int, len: int): void + asm(x b len) "STIX"; +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var re = x == 0 ? null : 100; @@ -36,7 +44,7 @@ fun test103(x: int?): (bool, bool, int) { } @method_id(104) -fun test104(x: int?) { +fun test104(x: MIntN) { var x2 = eq(x = 10); var ab = (x2, getNullableIntNull()); return (unwrap(ab.0) + (ab.1 == null ? -100 : ab.1!), ab.1); @@ -44,7 +52,7 @@ fun test104(x: int?) { @method_id(105) fun test105() { - var xy: (int?, int?) = (5, null); + var xy: (int?, MIntN) = (5, null); var ab = [1 ? [xy.0, xy.1] : null]; ab.0!.0 = intOr0(ab.0!.0); ab.0!.1 = intOr0(ab.0!.1); @@ -52,7 +60,7 @@ fun test105() { } global gTup106: tuple?; -global gInt106: int?; +global gInt106: MIntN; @method_id(106) fun test106() { @@ -60,8 +68,8 @@ fun test106() { gInt106! += 5; var int106: int? = 0; var gTup106 = createEmptyTuple(); - gTup106!.tuplePush(createEmptyTuple()); - (gTup106!.0 as tuple?)!.tuplePush(0 as int?); + gTup106!.push(createEmptyTuple()); + (gTup106!.0 as tuple?)!.push(0 as int?); tuplePush(mutate gTup106!, gInt106); tuplePush(mutate gTup106!.0, int106! += 1); return (gTup106 == null, null != gTup106, gTup106, gTup106!.0 as tuple?); @@ -74,7 +82,7 @@ fun test107() { b = b!.storeInt(3, 32); storeInt(mutate b!, 4, 32); (b! as builder).storeInt(5, 32); - return b!.getBuilderBitsCount(); + return b!.bitsCount(); } @method_id(108) @@ -91,7 +99,11 @@ fun test109() { return ([a, b] = [3, 4], a, b); } -fun main(x: int?, y: int?) { +fun main(x: int?, y: MIntN) { + __expect_type(x! == y!, "bool"); + __expect_type(5 != y!, "bool"); + __expect_type(y == null, "bool"); + __expect_type(true == (true as bool?)!, "bool"); } /** diff --git a/tolk-tester/tests/numbers-tests.tolk b/tolk-tester/tests/numbers-tests.tolk new file mode 100644 index 000000000..7a08740ba --- /dev/null +++ b/tolk-tester/tests/numbers-tests.tolk @@ -0,0 +1,35 @@ +const x1 = 0xFF; +const x2 = 0xAbcdEF01234561AC; +const x3 = 0x00BEC; +const x4 = -0xD70B; + +const d1 = 255; +const d2 = 12379813738877116844; +const d3 = 003052; +const d4 = -55051; + +const b1 = 0b11111111; +const b2 = 0b1010101111001101111011110000000100100011010001010110000110101100; +const b3 = 0b00101111101100; +const b4 = -0b1101011100001011; + +fun main() { + assert(x1 == d1 && x1 == b1, 100); + assert(x2 == d2 && x2 == b2, 100); + assert(x3 == d3 && x3 == b3, 100); + assert(x4 == d4 && x4 == b4, 100); + assert(x1 + x2 + x3 == b1 + b2 + b3, 100); + assert(-x2 == -d2 && -x2 == -b2, 100); + + return ( + 0b1010101010101010+0b00 == 43690, + 0b0 == 0, + -0b1010100001010101010-0b1+0b1 == -344746, + 0b00001 == 1, + -(-0b00101111101100) == 3052 + ); +} + +/** +@testcase | 0 | | -1 -1 -1 -1 -1 + */ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index e3c6bb6e8..79c837693 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -1,7 +1,10 @@ fun justTrue(): bool { return true; } +@noinline fun unary_minus_1(a: int, b: int, c: int): int{return -(a+b) *c;} +@noinline fun unary_minus_2(a: int, b: int, c: int): int{return(-(a+b))*c;} +@noinline fun unary_minus_3(a: int, b: int, c: int): int{return-((a+b) *c);} @@ -92,28 +95,22 @@ fun main() { @fif_codegen """ - unary_minus_1 PROC:<{ - // a b c - -ROT // c a b - ADD // c '3 + unary_minus_1() PROC:<{ // a b c + s0 s2 XCHG + ADD // c '3 NEGATE // c '4 - SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_2 PROC:<{ - // a b c - -ROT // c a b - ADD // c '3 + unary_minus_2() PROC:<{ // a b c + s0 s2 XCHG + ADD // c '3 NEGATE // c '4 - SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_3 PROC:<{ - // a b c - -ROT // c a b - ADD // c '3 - SWAP // '3 c - MUL // '4 + unary_minus_3() PROC:<{ // a b c + s0 s2 XCHG + ADD // c '3 + MUL // '4 NEGATE // '5 }> """ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk new file mode 100644 index 000000000..9c9acf821 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -0,0 +1,125 @@ +struct JustInt32 { + value: int32; +} + +struct Point { + x: int32; + y: int32; +} + +@method_id(101) +fun test1(value: int) { + var t: JustInt32 = { value }; + var c = t.toCell(); + var t2 = c.load(); + __expect_type(c, "Cell"); + __expect_type(t2, "JustInt32"); + if (0) { + var rawCell: cell = c; // Cell is assignable to raw cell + contract.setData(t.toCell()); // (for this purpose, since we don't have overloads) + } + + var t3 = Cell.load(c); + var t4 = JustInt32.fromCell(c); + var t5 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value, t4.value, t5.value); +} + +@method_id(102) +fun test2() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var s = c.tvmCell.beginParse(); + __expect_type(s.skipAny(), "slice"); + s.assertEnd(); + return true; +} + +@method_id(103) +fun test3() { + var yy = 20; + var p: Point = { x: 10, y: yy }; + var c = p.toCell(); // p is constant, storing its fields are joined into a single STI + var s = c.beginParse(); + s.skipAny(); // skipping is also merged to skip 64 bits + return s.remainingBitsCount(); +} + +@method_id(104) +fun test4() { + var s = stringHexToSlice("000000010000000200000003"); + var p = Point.fromSlice(s, {assertEndAfterReading: false}); // does not mutate s + return (p, s.remainingBitsCount()); +} + +@method_id(105) +fun test5() { + var s = stringHexToSlice("000000010000000200000003"); + var p: Point = s.loadAny(); // mutates s + return (p, s.remainingBitsCount()); +} + +@method_id(106) +fun test6() { + var b = beginCell().storeInt(1, 32); + var p: Point = { x: 10, y: 20 }; + b.storeAny(p); + var s = b.endCell().beginParse(); + return (s.remainingBitsCount(), b.bitsCount(), Point.fromSlice(s.getLastBits(64)), s.remainingBitsCount()); +} + +// this function is used in serialization codegen, it's not exposed to stdlib; +// it constructs "x{...}" slices for SDBEGINSQ, used for opcode matching; +// here we write some unit tests and fif codegen tests for it +fun slice.tryStripPrefix(mutate self, prefix: int, prefixLen: int): bool + builtin; + +@method_id(107) +fun test7(s: slice) { + if (s.tryStripPrefix(0x10, 32)) { return (1, s.remainingBitsCount()); } + if (s.tryStripPrefix(0x40, 31)) { return (2, s.remainingBitsCount()); } + if (s.tryStripPrefix(0xFF18, 16)) { return (3, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b1001, 8)) { return (4, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b01, 3)) { return (5, s.remainingBitsCount()); } + return (6, s.remainingBitsCount()); +} + +fun main(c: cell) { + c as Cell; + (c as Cell) as cell; + (c as Cell) as cell?; + __expect_type((c as Cell?)!, "Cell"); + c = c as Cell; // Cell implicitly converts to cell +} + +/** +@testcase | 101 | 10 | 10 10 10 10 +@testcase | 102 | | -1 +@testcase | 103 | | 0 +@testcase | 104 | | 1 2 96 +@testcase | 105 | | 1 2 32 +@testcase | 106 | | 96 96 10 20 96 +@testcase | 107 | x{00000080} | 2 1 +@testcase | 107 | x{09332} | 4 12 +@testcase | 107 | x{2} | 5 1 +@testcase | 107 | x{0234} | 6 16 + +@fif_codegen +""" + test3() PROC:<{ + NEWC + x{0000000a00000014} STSLICECONST // b + ENDC // c + CTOS // s + 64 LDU + NIP // s + SBITS // '17 + }> +""" + +@fif_codegen x{00000010} SDBEGINSQ +@fif_codegen b{0000000000000000000000001000000} SDBEGINSQ +@fif_codegen x{ff18} SDBEGINSQ +@fif_codegen x{09} SDBEGINSQ +@fif_codegen b{001} SDBEGINSQ + */ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk new file mode 100644 index 000000000..cb01fbfeb --- /dev/null +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -0,0 +1,759 @@ + +struct MaybeNothing {} +struct MaybeJust { value: T; } +type Maybe = MaybeNothing | MaybeJust; + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + +fun assert_slice_is_44_and_ref45(s: slice) { + assert(s.loadInt(32) == 44, 400); + var ref = s.loadRef().beginParse(); + assert(ref.loadInt(32) == 45, 400); + ref.assertEnd(); + s.assertEnd(); +} + +@inline_ref +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +@noinline +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* + value:int32 + = JustInt32; +*/ + +struct JustInt32 { + value: int32; +} + +/* + value:(Maybe int32) + = JustMaybeInt32; + */ + +struct JustMaybeInt32 { + value: int32?; +} + +/* + op:int32 + amount:Grams + = TwoInts32AndCoins; +*/ + +struct TwoInts32AndCoins { + op: int32; + amount: coins; +} + +/* + op:int32 + query_id:uint64 + = TwoInts32And64; +*/ + +struct TwoInts32And64 { + op: int32; + query_id: uint64; +} + +/* + op:int32 + query_id_ref: ^[uint64] + = TwoInts32AndRef64; +*/ + +struct TwoInts32AndRef64 { + op: int32; + query_id_ref: Cell; +} + +/* + op:int32 + query_id:(Maybe uint64) + demo_bool_field:Bool + = TwoInts32AndMaybe64; +*/ + +struct TwoInts32AndMaybe64 { + op: int32; + query_id: uint64?; + demo_bool_field: bool; +} + +/* + addr:MsgAddressInt + = JustAddress; +*/ + +struct JustAddress { + addr: address; +} + +/* + op:int32 + addr:MsgAddressExt + query_id:uint64 + = TwoInts32And64SepByAddress; +*/ + +struct TwoInts32And64SepByAddress { + op: int32; + addr_e: address; + query_id: uint64; +} + +/* + op:int32 + i8or256:(Either int8 int256) + = IntAndEitherInt8Or256; +*/ + +struct IntAndEitherInt8Or256 { + op: int32; + i8or256: int8 | int256; +} + +/* + query_id_ref:uint64 = Inner1; + i64_in_ref:int64 = Inner2; + + op:int32 + i32orRef:(Either int32 ^Inner2) + query_id_maybe_ref: (Maybe ^Inner1) + = IntAndEither32OrRef64; +*/ + +struct Inner1 { + query_id_ref: uint64; +} +struct Inner2 { + i64_in_ref: int64; +} + +struct IntAndEither32OrRef64 { + op: int32; + i32orRef: int32 | Cell; + query_id_maybe_ref: Cell?; +} + +/* + value:(Either int8 (Maybe int256)) + op:int32 + = IntAndEither8OrMaybe256; +*/ + +struct IntAndEither8OrMaybe256 { + value: Either; + op: int32; +} + +/* + value:(Either (Maybe int8) int256) + op:int32 + = IntAndEitherMaybe8Or256; +*/ + +struct IntAndEitherMaybe8Or256 { + value: Either; + op: int32; +} + +/* + value:(Maybe (Maybe int8)) + op:int32 + = IntAndMaybeMaybe8; +*/ + +struct IntAndMaybeMaybe8 { + value: Maybe>; + op: int32; +} + +/* + f1:bits8 + f2:bits3 + f3:(Maybe bits20) + f4:(Either bits100 bits200) + = SomeBytesFields; +*/ + +struct SomeBytesFields { + f1: bytes1; + f2: bits3; + f3: bits20?; + f4: bits100 | bits200; +} + +/* + op:int32 + rest:Cell + = IntAndRestInlineCell; +*/ + +struct IntAndRestInlineCell { + op: int32; + rest: RemainingBitsAndRefs; +} + +/* + op:int32 + rest:^Cell + = IntAndRestRefCell; +*/ + +struct IntAndRestRefCell { + op: int32; + rest: cell; +} + +/* + op:int32 + rest:(Either Cell ^Cell) + = IntAndRestEitherCellOrRefCell; +*/ + +struct IntAndRestEitherCellOrRefCell { + op: int32; + rest: RemainingBitsAndRefs | cell; +} + +/* + op:int32 + ref1m:(Maybe ^Cell) + ref2m:(Maybe ^Cell) + ref3:^Cell + ref4m32:(Maybe ^JustInt32) + query_id:int64 + = DifferentMaybeRefs; + */ + +struct DifferentMaybeRefs { + op: int32; + ref1m: cell?; + ref2m: dict; + ref3: cell; + ref4m32: Cell?; + query_id: int64; +} + +/* + ji:JustInt32 + jmi:JustMaybeInt32 + jiMaybe:(Maybe JustInt32) + jmiMaybe:(Maybe JustMaybeInt32) + = DifferentIntsWithMaybe; +*/ + +struct DifferentIntsWithMaybe { + ji: JustInt32; + jmi: JustMaybeInt32; + jiMaybe: JustInt32?; + jmiMaybe: JustMaybeInt32?; +} + +/* + ja1:JustAddress + ja2m:(Maybe JustAddress) + imm:IntAndMaybeMaybe8 + tis:TwoInts32And64SepByAddress + = DifferentMix1; +*/ + +@overflow1023_policy("suppress") +struct DifferentMix1 { + ja1: JustAddress; + ja2m: JustAddress?; + ext_nn: address; + imm: IntAndMaybeMaybe8; + tis: TwoInts32And64SepByAddress; +} + +/* + iae:^IntAndEither32OrRef64 + tic:TwoInts32AndCoins + rest:Cell + = DifferentMix2; +*/ + +struct DifferentMix2 { + iae: Cell; + tic: TwoInts32AndCoins; + rest: RemainingBitsAndRefs; +} + +/* + bod:(Either ^TwoInts32AndCoins ^JustInt32) + tim:(Maybe TwoInts32AndCoins) + pairm:(Maybe (Both int32 int64)) + = DifferentMix3; + */ + +struct DifferentMix3 { + bod: Cell | Cell; + tim: TwoInts32AndCoins?; + pairm: (int32, int64)?; +} + +/* + ui16: VarUInteger 16 + i16: VarInteger 16 + ui32: VarUInteger 32 + i32: VarInteger 32 + */ + +struct WithVariadicInts { + ui16: varuint16; + i16: varint16; + ui32: varuint32; + i32: varint32; +} + +/* + Test MIN_INT and MAX_INT + */ + +struct EdgeCaseInts { + maxUint: uint256 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + maxInt: int257 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + minInt: int257 = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1; +} + +/* + Test: write with builder, read with other struct + (type `builder` is available for writing, but not for reading) + */ + +struct WriteWithBuilder { + f1: int32; + rest: builder; +} + +struct WriteWithSlice { + f1: int32; + rest: slice; +} + +struct ReadWrittenWithBuilder { + f1: int32; + someInt: uint32; + someCell: cell?; +} + +struct ReadWriteRest { + f1: int32; + f2: coins; + rest: T; +} + +struct Tail224 { + ji: JustInt32, + addr: address, + ref1: Cell?, + ref2: Cell, +} + +type ReadRest_Remaining = ReadWriteRest; + +struct ReadWriteMid { + f1: int32; + mid: T; + f3: coins; +} + +struct WithTwoRestFields { + i32: int32; + rest1: RemainingBitsAndRefs; + rest2: RemainingBitsAndRefs; +} + + + +// --------------------------------------------- + + +@method_id(200) +fun test_JustInt32_1() { + var b = beginCell(); + b.storeAny(JustInt32{ value: 123 }); + b.storeAny(JustInt32{ value: 456 }); + return b; +} + +@method_id(201) +fun test_JustInt32() { + run({ value: 255}, stringHexToSlice("000000FF")); + var s = stringHexToSlice("0000007b000001c8"); + return (s.loadAny(), s.loadAny()); +} + +@method_id(202) +fun test_JustMaybeInt32() { + run({ value: 255 }, stringHexToSlice("8000007FC_")); + run({ value: null }, stringHexToSlice("4_")); + return true; +} + +@method_id(203) +fun test_TwoInts32AndCoins() { + run({ op: 123, amount: 0 }, stringHexToSlice("0000007B0")); + run({ op: 123, amount: 1000000000 }, stringHexToSlice("0000007B43B9ACA00")); + return true; +} + +@method_id(204) +fun test_TwoInts32And64() { + run({ op: 123, query_id: 255 }, stringHexToSlice("0000007B00000000000000FF")); + return true; +} + +@method_id(205) +fun test_TwoInts32AndRef64() { + run({ op: 123, query_id_ref: { tvmCell: beginCell().storeUint(255,64).endCell() } }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); + return true; +} + +@method_id(206) +fun test_TwoInts32AndMaybe64() { + run({ op: 123, query_id: 255, demo_bool_field: true }, stringHexToSlice("0000007B800000000000007FE_")); + run({ op: 123, query_id: null, demo_bool_field: true }, stringHexToSlice("0000007B6_")); + run({ op: 123, query_id: null, demo_bool_field: false }, stringHexToSlice("0000007B2_")); + return true; +} + +@method_id(207) +fun test_JustAddress() { + run({ addr: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, stringHexToSlice("80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_")); + return true; +} + +@method_id(208) +fun test_TwoInts32And64SepByAddress() { + run({ op: 123, addr_e: makeExternalAddress(70, 10), query_id: 255 } , stringHexToSlice("0000007B41423000000000000007FC_")); + run({ op: 123, addr_e: makeExternalAddress(666, 20), query_id: 254 } , stringHexToSlice("0000007B4280053400000000000001FD_")); + run({ op: 123, addr_e: createAddressNone(), query_id: 253 } , stringHexToSlice("0000007B000000000000003F6_")); + return true; +} + +@method_id(209) +fun test_IntAndEitherInt8Or256() { + run({ op: 123, i8or256: 80 as int8 }, stringHexToSlice("0000007B284_")); + run({ op: 123, i8or256: 65535 as int256 }, stringHexToSlice("0000007B8000000000000000000000000000000000000000000000000000000000007FFFC_")); + return true; +} + +@method_id(210) +fun test_IntAndEither32OrRef64() { + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007BE_").appendRef(stringHexToSlice("000000000000022B")).appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: null }, stringHexToSlice("0000007BA_").appendRef(stringHexToSlice("000000000000022B"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007B00000115E_").appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: null }, stringHexToSlice("0000007B00000115A_")); + return IntAndEither32OrRef64.fromSlice(stringHexToSlice("0000007B00000115A_")); +} + +@method_id(211) +fun test_IntAndEither8OrMaybe256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("320000003DC_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("C0000000000000000000000000000000000000000000000000000000000009C40000001EE_")); + run({ value: EitherRight { value: null }, op: 123 }, stringHexToSlice("8000001EE_")); + return IntAndEither8OrMaybe256.fromSlice(stringHexToSlice("8000001EE_")); +} + +@method_id(212) +fun test_IntAndEitherMaybe8Or256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("590000001EE_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("80000000000000000000000000000000000000000000000000000000000013880000003DC_")); + run({ value: EitherLeft { value: null }, op: 123 }, stringHexToSlice("0000001EE_")); + return IntAndEitherMaybe8Or256.fromSlice(stringHexToSlice("0000001EE_")); +} + +@method_id(213) +fun test_SomeBytesFields() { + run({ f1: stringHexToSlice("A4") as bytes1, f2: stringHexToSlice("7_") as bits3, f3: null, f4: stringHexToSlice("BBA87684B3DAA58C0FCC75230C4302C9D156102139D631FF56") as bits200 }, stringHexToSlice("A46DDD43B4259ED52C607E63A9186218164E8AB08109CEB18FFAB4_")); + run({ f1: stringHexToSlice("E6") as bytes1, f2: stringHexToSlice("D_") as bits3, f3: stringHexToSlice("2531C") as bits20, f4: stringHexToSlice("927E88FAB2D327D9468547217") as bits100 }, stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")); + return SomeBytesFields.fromSlice(stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")).f4 is bits100; +} + +@method_id(214) +fun test_IntAndMaybeMaybe8() { + run({ value: MaybeJust { value: MaybeJust { value: 88 } }, op: 123 }, stringHexToSlice("D60000001EE_")); + run({ value: MaybeJust { value: MaybeNothing{} }, op: 123 }, stringHexToSlice("8000001EE_")); + + val t1 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("D60000001EE_")); // (88) 123 + val v1 = t1.value; + __expect_type(v1, "Maybe>"); + assert(v1 is MaybeJust && v1.value is MaybeJust && v1.value.value == 88 && t1.op == 123, 400); + __expect_type(v1, "MaybeJust>"); + __expect_type(v1.value, "MaybeJust"); + __expect_type(v1.value.value, "int8"); + val t2 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("8000001EE_")); // (()) 123 + val v2 = t2.value; + assert(v2 is MaybeJust && v2.value is MaybeNothing && t2.op == 123, 400); + __expect_type(v2, "MaybeJust>"); + __expect_type(v2.value, "MaybeNothing"); + + return true; +} + +@method_id(215) +fun test_IntAndRestInlineCell() { + run({ op: 123, rest: generateSlice_44_with_ref45() }, stringHexToSlice("0000007B0000002C").appendRef(stringHexToSlice("0000002D"))); + return true; +} + +@method_id(216) +fun test_IntAndRestRefCell() { + run({ op: 123, rest: generateCell_44_with_ref45() }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + return true; +} + +@method_id(217) +fun test_IntAndRestEitherCellOrRefCell() { + val input1: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateCell_44_with_ref45() }; + var s1 = input1.toCell().beginParse(); + s1.assertEqDeeply(stringHexToSlice("0000007BC_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s1).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + val input2: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateSlice_44_with_ref45() }; + var s2 = input2.toCell().beginParse(); + s2.assertEqDeeply(stringHexToSlice("0000007B000000164_").appendRef(stringHexToSlice("0000002D"))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s2).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + return true; +} + +@method_id(218) +fun test_DifferentMaybeRefs() { + run({ op: 123, ref1m: null, ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007B00000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 999}.toCell(), query_id: 456}, stringHexToSlice("0000007BA0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E7"))); + run({ op: 123, ref1m: null, ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 998}.toCell(), query_id: 456}, stringHexToSlice("0000007B60000000000000391_").appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E6"))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: generateCell_44_with_ref45(), ref3: beginCell().endCell(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007BC0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("")); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 997}.toCell(), query_id: 456}, stringHexToSlice("0000007BE0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E5"))); + return true; +} + +@method_id(219) +fun test_DifferentIntsWithMaybe() { + run({ ji: { value: 44 }, jmi: { value: 45 }, jiMaybe: null, jmiMaybe: null }, stringHexToSlice("0000002C800000169_")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: { value: 45 }, jmiMaybe: { value: null } }, stringHexToSlice("0000002C4000000B6")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: null, jmiMaybe: { value: 46 } }, stringHexToSlice("0000002C30000002E")); + return DifferentIntsWithMaybe.fromSlice(stringHexToSlice("0000002C30000002E")); +} + +@method_id(220) +fun test_DifferentMix1() { + run({ ja1: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ja2m: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ext_nn: makeExternalAddress(1234,30), imm: { op: 78, value: MaybeJust { value: MaybeJust { value: 78 } } }, tis: { op: 123, query_id: 889128, addr_e: makeExternalAddress(1234, 80) } }, + stringHexToSlice("80122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62180122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62087800004D2D3800000138000001ED2800000000000000000269000000000006C8944_")); + run({ ja1: { addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") }, ja2m: null, ext_nn: makeExternalAddress(0,3), imm: { op: 99, value: MaybeJust { value: MaybeJust { value: 99 } } }, tis: { op: 1234, query_id: 889129, addr_e: createAddressNone() } }, + stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + + val o = DifferentMix1.fromSlice(stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + return (o.ja1.addr.getWorkchainAndHash(), o.ja2m, o.imm, o.tis.op, o.tis.addr_e.isNone() ? null : 0, o.tis.query_id, (o.ext_nn as slice).remainingBitsCount()); +} + +@method_id(221) +fun test_DifferentMix2() { + run({ iae: IntAndEither32OrRef64{ op: 777, i32orRef: 2983, query_id_maybe_ref: null }.toCell(), tic: { op: 123, amount: 829290000 }, rest: generateSlice_44_with_ref45() }, + stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val r1 = DifferentMix2.fromSlice(stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val iae1 = r1.iae.load(); + assert(iae1.i32orRef is int32, 400); + assert(iae1.i32orRef == 2983, 400); + assert(iae1.query_id_maybe_ref == null, 400); + assert_slice_is_44_and_ref45(r1.rest); + + run({ iae: { tvmCell: IntAndEither32OrRef64{ op: 778, i32orRef: { tvmCell: Inner2{ i64_in_ref: 9919992 }.toCell() }, query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell() }, tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, + stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + var r2 = DifferentMix2.fromSlice(stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + val iae2 = r2.iae.load(); + assert(iae2.i32orRef is Cell, 400); + __expect_type(iae2.i32orRef.load(), "Inner2"); + assert(iae2.i32orRef.load().i64_in_ref == 9919992, 400); + assert(iae2.query_id_maybe_ref != null, 400); + __expect_type(iae2.query_id_maybe_ref.load(), "Inner1"); + assert(iae2.query_id_maybe_ref.load().query_id_ref == 889477, 400); + assert(r2.tic.amount == 500000, 400); + r2.rest.assertEnd(); + + return true; +} + +@method_id(222) +fun test_DifferentMix3() { + run({ bod: TwoInts32AndCoins{op:123, amount:80000}.toCell(), tim: {op: 456, amount:0}, pairm: null }, stringHexToSlice("4000007201_").appendRef(stringHexToSlice("0000007B3013880"))); + run({ bod: TwoInts32AndCoins{op:124, amount:10}.toCell(), tim: null, pairm: (100000,100000) }, stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + run({ bod: JustInt32{value:255}.toCell(), tim: null, pairm: (90,90) }, stringHexToSlice("A000000B400000000000000B5_").appendRef(stringHexToSlice("000000FF"))); + run({ bod: JustInt32{value:510}.toCell(), tim: {op: 567, amount:9392843922}, pairm: (81923,81923) }, stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + + val o4 = DifferentMix3.fromSlice(stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + val o2 = DifferentMix3.fromSlice(stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + return ( + (o4.bod is Cell) ? o4.bod.load().value as int : -1, o4.tim, o4.pairm, + 777, + o2.bod is Cell, o2.bod is Cell, o2.tim, o2.pairm, + ); +} + +@method_id(223) +fun test_WriteWithBuilderReadWithOther() { + var b = beginCell().storeUint(55, 32).storeMaybeRef(null); + var w: WriteWithBuilder = { f1: 10, rest: b }; + var w2: WriteWithSlice = { f1: 10, rest: stringHexToSlice("FFFF") }; + assert(w2.toCell().beginParse().skipBits(32).loadUint(16) == 0xFFFF, 100); + return ReadWrittenWithBuilder.fromCell(w.toCell()); +} + +@method_id(224) +fun test_RestIsBuilderOrRemaining() { + var remainingB = beginCell() + .storeUint(5, 32) + .storeAddress(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeMaybeRef(JustInt32 { value: 123 }.toCell()) + .storeRef(JustAddress { addr: createAddressNone() }.toCell()); + var w: ReadWriteRest = { + f1: 60, + f2: ton("0.05"), + rest: remainingB + }; + var r: ReadRest_Remaining = w.toCell().beginParse().loadAny(); + __expect_type(r.rest, "RemainingBitsAndRefs"); + var rest = Tail224.fromSlice(r.rest); + return (r.f1, r.f2, rest.ji.value, rest.addr.getWorkchain(), rest.ref1!.load(), rest.ref2.load().addr.isNone()); +} + +@method_id(225) +fun test_MidIsBuilderOrBitsN() { + var bits40_b = beginCell().storeSlice(stringHexToSlice("0000FFFF00")); + var typedCell = ReadWriteMid { + f1: 5, + mid: bits40_b, + f3: ton("0.05") + }.toCell(); + __expect_type(typedCell, "Cell>"); + var r = ReadWriteMid.fromCell(typedCell); + var mid = r.mid as slice; + return (r.f1, mid.remainingBitsCount(), mid.loadAny(), mid.remainingBitsCount(), r.f3); +} + +@method_id(226) +fun test_MultipleRemainers() { + val m = WithTwoRestFields.fromSlice(stringHexToSlice("00000001FFFF")); + return (m.i32, m.rest1.remainingBitsCount(), m.rest2.remainingBitsCount()); +} + +@method_id(227) +fun test_mutatingRemainder() { + var cs = stringHexToSlice("00000001FFFF"); + val obj = cs.loadAny(); + cs.assertEnd(); + return obj.rest.remainingBitsCount(); +} + +@method_id(228) +fun test_VariadicIntegers() { + var t: WithVariadicInts = { + ui16: (1 << 120) - 1, + i16: -(1 << 119), + ui32: (1 << 248) - 1, + i32: -(1 << 247), + }; + run(t, stringHexToSlice("ffffffffffffffffffffffffffffffff800000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe000000000000000000000000000000000000000000000000000000000000020_")); + run({ui16: 0, i16: 0, ui32: 0, i32: 0}, stringHexToSlice("000020_")); + + var c = t.toCell().load(); + return (c.ui16 == t.ui16) & (c.i16 == t.i16) & (c.ui32 == t.ui32) & (c.i32 == t.i32) +} + +@method_id(229) +fun test_EdgeCaseIntegers() { + var edge: EdgeCaseInts = {}; + var manual = beginCell().storeUint(edge.maxUint, 256).storeInt(edge.maxInt, 257).storeInt(edge.minInt, 257); + return edge.toCell().hash() == manual.endCell().hash() +} + +fun main() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var t2 = c.load(); + var t3 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value); +} + +/** +@testcase | 0 | | 10 10 +@testcase | 200 | | BC{00100000007b000001c8} +@testcase | 201 | | 123 456 +@testcase | 202 | | -1 +@testcase | 203 | | -1 +@testcase | 204 | | -1 +@testcase | 205 | | -1 +@testcase | 206 | | -1 +@testcase | 207 | | -1 +@testcase | 208 | | -1 +@testcase | 209 | | -1 +@testcase | 210 | | 123 555 46 (null) +@testcase | 211 | | (null) typeid-4 123 +@testcase | 212 | | (null) typeid-5 123 +@testcase | 213 | | -1 +@testcase | 214 | | -1 +@testcase | 215 | | -1 +@testcase | 216 | | -1 +@testcase | 217 | | -1 +@testcase | 218 | | -1 +@testcase | 219 | | 44 (null) (null) 46 typeid-15 +@testcase | 220 | | 0 0 (null) 99 typeid-8 typeid-9 99 1234 (null) 889129 14 +@testcase | 221 | | -1 +@testcase | 222 | | 510 567 9392843922 typeid-18 81923 81923 typeid-19 777 0 -1 (null) (null) 0 100000 100000 typeid-19 +@testcase | 223 | | 10 55 (null) +@testcase | 224 | | 60 50000000 5 9 123 -1 +@testcase | 225 | | 5 40 65535 8 50000000 +@testcase | 226 | | 1 16 0 +@testcase | 227 | | 16 +@testcase | 228 | | -1 +@testcase | 229 | | -1 + */ diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk new file mode 100644 index 000000000..793f714b9 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -0,0 +1,284 @@ + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline_ref +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + + +@noinline +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* +single_prefix32#87654321 + amount1:Grams + amount2:Grams + = MsgSinglePrefix32; +*/ + +struct(0x87654321) MsgSinglePrefix32 { + amount1: coins; + amount2: coins; +} + +/* +single_prefix48#876543211234 + amount:(Either Grams uint64) + = MsgSinglePrefix48; +*/ + +struct(0x876543211234) MsgSinglePrefix48 { + amount: coins | uint64; +} + +/* +counterIncrement#12345678 + counter_id:int8 + inc_by:int32 + = MsgCounter1; +*/ + +struct(0x12345678) CounterIncrement { + counter_id: int8; + inc_by: int32; +} + +/* +counterDecrement#23456789 + counter_id:int8 + dec_by:int32 + = MsgCounter1; +*/ + +struct(0x23456789) CounterDecrement { + counter_id: int8; + dec_by: int32; +} + +/* +counterReset0#34567890 + counter_id:int8 + = MsgCounter1; +*/ + +struct(0x34567890) CounterReset0 { + counter_id: int8; +} + +/* +counterResetTo#00184300 + counter_id:int8 + initial_value:int64 + = MsgCounter1; +*/ + +struct(0x00184300) CounterResetTo { + counter_id: int8; + initial_value: int64; +} + +type MsgCounter1 = + | CounterIncrement + | CounterDecrement + | CounterReset0 + | CounterResetTo + +/* +bodyPayload1$001 + should_forward:Bool + n_times:int32 + content:Cell + = BodyPayload; +*/ + +struct(0b001) BodyPayload1 { + should_forward: bool; + n_times: int32; + content: RemainingBitsAndRefs; +} + +/* +bodyPayload2$01 + master_id:int8 + owner_address:MsgAddressInt + = BodyPayload; +*/ + +struct(0b01) BodyPayload2 { + master_id: int8; + owner_address: address; +} + +type BodyPayload = BodyPayload1 | BodyPayload2 + +/* +sayHiAndGoodbye#89 + dest_addr:(Maybe MsgAddressInt) + body:BodyPayload + = MsgExternal1; +*/ + +struct(0x89) SayHiAndGoodbye { + dest_addr: address?; + body: BodyPayload; +} + +/* +sayStoreInChain#0013 + in_masterchain:Bool + contents:^BodyPayload + = MsgExternal1; + */ + +struct(0x0013) SayStoreInChain { + in_masterchain: bool; + contents: Cell; +} + +type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain + +/* +transferParams1#794 + dest_int:MsgAddressInt + amount:Grams + dest_ext:MsgAddressExt + = TransferParams; +*/ + +struct(0x794) TransferParams1 { + dest_int: address; + amount: coins; + dest_ext: address; +} + +/* +transferParams2#9 + intVector:(Both int32 (Both (Maybe Grams) uint64)) + needs_more:^Bit + = TransferParams; + */ + +struct(0x9) TransferParams2 { + intVector: (int32, coins?, uint64); + needs_more: Cell; +} + +type TransferParams = TransferParams1 | TransferParams2; + +/* +_#FB3701FF + params:(Either TransferParams ^TransferParams) + = MsgTransfer; + */ + +struct(0xFB3701FF) MsgTransfer { + params: Either>; +} + + + +// --------------------------------------------- + + +@method_id(201) +fun test_MsgSinglePrefix32() { + run({ amount1: 80, amount2: 800000000 }, stringHexToSlice("8765432115042FAF0800")); + + return MsgSinglePrefix32.fromSlice(stringHexToSlice("8765432115042FAF0800")); +} + +@method_id(202) +fun test_MsgSinglePrefix48() { + run({ amount: 80 as uint64 }, stringHexToSlice("87654321123480000000000000284_")); + run({ amount: 800000000 as coins }, stringHexToSlice("876543211234217D784004_")); + + var o = MsgSinglePrefix48.fromSlice(stringHexToSlice("876543211234217D784004_")); + return (o, 777, o.amount is coins, o.amount is uint64); +} + +@method_id(203) +fun test_MsgCounter1() { + run(CounterIncrement{ counter_id: 123, inc_by: 78 }, stringHexToSlice("123456787B0000004E")); + run(CounterDecrement{ counter_id: 0, dec_by: -38 }, stringHexToSlice("2345678900FFFFFFDA")); + run(CounterReset0{ counter_id: 0 }, stringHexToSlice("3456789000")); + run(CounterResetTo{ counter_id: 0, initial_value: 29874329774732 }, stringHexToSlice("001843000000001B2BA8D06A8C")); + + val o = MsgCounter1.fromSlice(stringHexToSlice("3456789000")); + return (o, 777, o is CounterReset0, o is CounterIncrement); +} + +@method_id(204) +fun test_MsgExternal1() { + run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); + run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); + run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); + run(SayStoreInChain{ in_masterchain: true, contents: { tvmCell: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell()} }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); + run(SayStoreInChain{ in_masterchain: false, contents: { tvmCell: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell()} }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + + val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + assert(o is SayStoreInChain, 400); + val contents = o.contents.load(); + return (contents is BodyPayload2 && contents.owner_address == address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), contents is BodyPayload1, contents is BodyPayload2); +} + +@method_id(205) +fun test_MsgTransfer() { + run({ params: EitherLeft { value: TransferParams1 { dest_int: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), amount: 80000000, dest_ext: makeExternalAddress(1234,80) } } }, stringHexToSlice("FB3701FF3CA4FF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E404C4B4004A0000000000000000009A5_")); + run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: {tvmCell: beginCell().storeBool(true).endCell()} } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); + run({ params: EitherRight { value: { tvmCell: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); + run({ params: EitherRight { value: { tvmCell: TransferParams2{ intVector: (123, null, 0), needs_more: {tvmCell: beginCell().storeBool(false).endCell()} }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + + val o = MsgTransfer.fromSlice(stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + return ( + o.params is EitherLeft, o.params is EitherRight, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams1, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams2, + ); +} + + +fun main() {} + +/** +@testcase | 201 | | 80 800000000 +@testcase | 202 | | 800000000 17 777 -1 0 +@testcase | 203 | | (null) 0 typeid-3 777 -1 0 +@testcase | 204 | | -1 0 -1 +@testcase | 205 | | 0 -1 0 -1 + */ diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk new file mode 100644 index 000000000..08a1b4328 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -0,0 +1,163 @@ + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + + +@noinline +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +type Union_8_16_32 = int8 | int16 | int32; + +fun ans101_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans101_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans101_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +@method_id(101) +fun test1() { + run(15 as int8, ans101_8()); + run(15 as int16, ans101_16()); + run(15 as int32, ans101_32()); + + return 0; +} + + +type Union_8_16_32_n = int8 | null | int16 | int32; + +fun ans102n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans102n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans102n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans102n_null(): slice asm "b{0} PUSHSLICE"; + +@method_id(102) +fun test2() { + run(15 as int8, ans102n_8()); + run(15 as int16, ans102n_16()); + run(15 as int32, ans102n_32()); + run(null, ans102n_null()); + + return 0; +} + + +struct Test4_8 { a: int8 } +struct Test4_16 { a: int16 } +struct Test4_32 { a: int32 } + +type UnionStructs_8_16_32 = int8 | Test4_16 | Test4_32; +type UnionStructs_8_16_32_n = Test4_8 | null | int16 | int32; + +fun ans104_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans104_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans104_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +fun ans104n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans104n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans104n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans104n_null(): slice asm "b{0} PUSHSLICE"; + +@method_id(104) +fun test4() { + // when mixing primitives and structs with no opcode, it's like mixing primitives + run(15, ans104_8()); + run(Test4_16{a:15}, ans104_16()); + run(Test4_32{a:15}, ans104_32()); + // with null — the same behavior + run({a:15}, ans104n_8()); + run(15 as int16, ans104n_16()); + run(15 as int32, ans104n_32()); + run(null, ans104n_null()); + + return 0; +} + + +type U105 = int8 | int32 | int64; // auto-prefixes 0b00 0b01 0b10 + +fun invalid105_slice(): slice asm "b{1100} PUSHSLICE"; // prefix 0b11 doesn't exist + +@method_id(105) +fun test5() { + try { + var u = U105.fromSlice(invalid105_slice(), {throwIfOpcodeDoesNotMatch: 9}); + return (u is int8) ? 8 : -8; + } catch (excode) { + return excode; + } +} + + +type U106 = int8 | int16 | int32 | int64; // exhaustive prefixes, checked via codegen + +fun s106_int16(): slice asm "b{010000000000001111} PUSHSLICE"; + +@method_id(106) +fun test6(): int16 { + var u = U106.fromSlice(s106_int16()); + if (u is int16) { return u; } + else { return -1; } +} + + +fun main() {} + +/** +@testcase | 101 | | 0 +@testcase | 102 | | 0 +@testcase | 104 | | 0 +@testcase | 105 | | 9 +@testcase | 106 | | 15 + +@fif_codegen +""" + test6() PROC:<{ // + b{010000000000001111} PUSHSLICE // s + b{00} SDBEGINSQ // s '8 + IF:<{ // s + 8 LDI // '12 s + 42 PUSHINT // 'USlot1 s 'UTag=42 + }>ELSE<{ // s + b{01} SDBEGINSQ // s '8 + IF:<{ // s + 16 LDI // '17 s + 44 PUSHINT // 'USlot1 s 'UTag=44 + }>ELSE<{ // s + b{10} SDBEGINSQ // s '8 + IF:<{ // s + 32 LDI // '22 s + 46 PUSHINT // 'USlot1 s 'UTag=46 + }>ELSE<{ // s + b{11} SDBEGINSQ // s '8 + IFNOTJMP:<{ // s + 63 THROW + }> + 64 LDI // '27 s + 48 PUSHINT // 'USlot1 s 'UTag=48 + }> + }> + }> + SWAP // 'USlot1 'UTag s + ENDS // u.USlot1 u.UTag + 44 EQINT // u.USlot1 '29 + IFJMP:<{ // u.USlot1 + }> // u.USlot1 + DROP // + -1 PUSHINT // '31=-1 + }> +""" + */ + diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk new file mode 100644 index 000000000..97b740b48 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -0,0 +1,220 @@ +struct JustInt32 { + value: int32; +} + +type MaybeInt32 = int32?; +type MaybeCell = cell?; +type Int16Or32 = int16 | int32; + +struct JustMaybeInt32 { + value: MaybeInt32; +} + + +@method_id(101) +fun test1() { + return ( + int32.estimatePackSize(), + uint64.estimatePackSize(), + int1.estimatePackSize(), + bool.estimatePackSize(), + RemainingBitsAndRefs.estimatePackSize(), + coins.estimatePackSize(), + bits6.estimatePackSize(), + bytes8.estimatePackSize(), + varint32.estimatePackSize(), + ); +} + +@method_id(102) +fun test2() { + return ( + JustInt32.estimatePackSize(), + JustMaybeInt32.estimatePackSize(), + MaybeInt32.estimatePackSize(), + Int16Or32.estimatePackSize(), + ); +} + +struct Test3_1 { + f1: JustInt32, + f2: JustMaybeInt32, + f3: JustMaybeInt32, + f4: JustInt32 | uint100, +} + +struct Test3_2 { + f1: Test3_1; + f2: Test3_1?; + f3: null | Test3_1 | null; +} + +@method_id(103) +fun test3() { + return (Test3_1.estimatePackSize(), Test3_2.estimatePackSize()); +} + +struct Test4_1 { + f1: address; + f2: address; +} + +struct Test4_2 { + f1: address?; + f2: address?; +} + +@overflow1023_policy("suppress") +struct Test4_3 { + f: (Test4_1, Test4_2); +} + +@method_id(104) +fun test4() { + return (Test4_1.estimatePackSize(), Test4_2.estimatePackSize(), Test4_3.estimatePackSize()); +} + +struct Test5_1 { + f1: cell; + f2: cell?; + f3: Cell; + f4: Cell?; +} + +struct Test5_2 { + f1: Cell?; + f2: Cell?; + f3: Cell>?; + f4: Cell?; + f5: Cell?; + rest: RemainingBitsAndRefs; +} + +struct Test5_3 { + f1: int8 | Cell; + f2: bytes2 | Cell; + f3: Cell | cell; + f4: Cell | coins; +} + +@method_id(105) +fun test5() { + return (Test5_1.estimatePackSize(), Test5_2.estimatePackSize(), Test5_3.estimatePackSize()); +} + +struct(0x00112233) Test6_1 { + f1: int32; +} + +struct(0b0010) Test6_2 { + f1: int32?; + f2: Cell; +} + +type Test6_or = Test6_1 | Test6_2; + +@method_id(106) +fun test6() { + return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); +} + +struct(0x1020) Test7_1; +struct(0x1030) Test7_2; +struct(0x1040) Test7_3; + +type Test7_or = | Test7_1 | Test7_2 | Test7_3 + +@method_id(107) +fun test7() { + assert((Test7_1{} as Test7_or).toCell().beginParse().remainingBitsCount() == Test7_or.estimatePackSize().0, 400); + return (Test7_1.estimatePackSize(), Test7_or.estimatePackSize()); +} + +struct(0x10) Inner8_1 { + ea: address; +} +struct(0b1) CellInner8_1 { + ref: Cell; +} +struct Inner8_2 { + t: (bits32, int1, coins?); +} + +@overflow1023_policy("suppress") +struct Test8 { + f1: Inner8_1; + f2: Inner8_2; + f3: Inner8_1?; + f4: Inner8_2?; + f5: Inner8_1 | CellInner8_1; +} + +@method_id(108) +fun test8() { + return (Inner8_1.estimatePackSize(), Inner8_2.estimatePackSize(), Test8.estimatePackSize()); +} + +struct Test9_bits2 { f: bits2; } +struct Test9_bits4 { f: bits4; } + +type Test9_f1 = int32 | int64 | int128; // auto-generated 2-bit prefix +type Test9_f2 = int32 | Inner8_2; // auto-generated 1-bit prefix (Either) +type Test9_f3 = bits1 | Test9_bits2 | bits3 | bits4 | bits5; // auto-generated 3-bit prefix +type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 0 / 100/101/110/111 + +@method_id(109) +fun test9() { + return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); +} + +struct Test10_1 { + a: int32; + b: builder; // unpredictable +} + +type Test10_2 = (Test10_1, bool?, RemainingBitsAndRefs); + +@method_id(110) +fun test10() { + return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); +} + +struct Test11_1 { + data: bits1022; + next: Cell?; +} + +struct Test11_2 { + self1: Cell; + self2: Cell; +} + +@method_id(111) +fun test11() { + return (Test11_1.estimatePackSize(), Test11_2.estimatePackSize()); +} + +@method_id(120) +fun test20() { + return (Test7_1 .getDeclaredPackPrefixLen(), Test7_1 .getDeclaredPackPrefix(), + CellInner8_1.getDeclaredPackPrefixLen(), CellInner8_1.getDeclaredPackPrefix()); +} + +fun main() { + __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); +} + +/** +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] [ 5 253 0 0 ] +@testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] +@testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] +@testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] +@testcase | 105 | | [ 2 2 2 4 ] [ 5 9999 0 9 ] [ 4 152 1 4 ] +@testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] +@testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] +@testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] +@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 1 7 0 0 ] +@testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] +@testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] +@testcase | 120 | | 16 4128 1 1 + */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk new file mode 100644 index 000000000..364d026a2 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -0,0 +1,219 @@ +struct (0x01) CounterIncrement { byValue: int8; } +struct (0x03) CounterDecrementBy1 {} +type CounterMsg = CounterIncrement | CounterDecrementBy1; + +struct SomeBytesFields { + f1: bytes1; +} + +@method_id(101) +fun test1() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("880f"), {throwIfOpcodeDoesNotMatch: 101}).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(102) +fun test2() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("890f")).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(103) +fun test3() { + var cc: Cell = { + tvmCell: beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() + }; + return cc.load({assertEndAfterReading: false}); +} + +@method_id(104) +fun test4() { + try { + var msg = CounterMsg.fromSlice(stringHexToSlice("88"), {throwIfOpcodeDoesNotMatch: 104, assertEndAfterReading: false}); + __expect_type(msg, "CounterMsg"); + return (-1, msg is CounterIncrement); + } catch (excno) { + return (excno, null); + } +} + +@method_id(105) +fun test5() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell().hash() & 0xFFFF; +} + +@method_id(106) +fun test6() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNValidation: true}).hash() & 0xFFFF; +} + +@method_id(107) +fun test7(believe: bool) { + try { + return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNValidation: believe}).hash() & 0xFFFF; + } catch (excno) { + return excno; + } +} + +@method_id(108) +fun test8() { + return CounterIncrement.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF + }).byValue; +} + +@method_id(109) +fun test9() { + return CounterMsg.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF, + assertEndAfterReading: false, + }) is CounterIncrement; +} + +@method_id(110) +fun test10() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + try { + var b = c.load(); + return -(b.f1 as slice).remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(111) +fun test11() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + var b = c.load({assertEndAfterReading: false}); + return -(b.f1 as slice).remainingBitsCount(); +} + +fun main(){} + +/** +@testcase | 101 | | 101 +@testcase | 102 | | 63 +@testcase | 103 | | 9 +@testcase | 104 | | 104 (null) +@testcase | 105 | | 36896 +@testcase | 106 | | 36896 +@testcase | 107 | -1 | 2142 +@testcase | 107 | 0 | 9 +@testcase | 108 | | 15 +@testcase | 109 | | -1 +@testcase | 110 | | 9 +@testcase | 111 | | -8 + +@fif_codegen +""" +x{880f} PUSHSLICE +x{01} SDBEGINSQ +101 THROWIFNOT +""" + +@fif_codegen +""" +x{890f} PUSHSLICE +x{01} SDBEGINSQ +63 THROWIFNOT +8 LDI +ENDS +RETALT +""" + +@fif_codegen +""" + test3() PROC:<{ + x{0109ab} PUSHSLICE + NEWC + STSLICE + ENDC + CTOS + x{01} SDBEGINSQ + 63 THROWIFNOT + 8 PLDI + }> +""" + +@fif_codegen +""" +IF:<{ + DROP + 139 PUSHINT +}>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ + 104 THROW + }> + 140 PUSHINT +}> +""" + +@fif_codegen +""" + test5() PROC:<{ + x{11} PUSHSLICE + NEWC + OVER + SBITREFS + 9 THROWIF + 8 EQINT + 9 THROWIFNOT + STSLICE +""" + +@fif_codegen +""" + test6() PROC:<{ + x{11} PUSHSLICE + NEWC + STSLICE + ENDC + HASHCU +""" + +@fif_codegen +""" + test8() PROC:<{ // + x{010f} PUSHSLICE // '0 + 16 PUSHPOW2DEC // s '2=65535 + SWAP // '2=65535 s + x{01} SDBEGINSQ // '2=65535 s '4 + s1 s2 XCHG // s '2=65535 '4 + THROWANYIFNOT // s + 8 LDI // '9 s + ENDS // '9 + }> +""" + +@fif_codegen +""" + test9() PROC:<{ + x{010f} PUSHSLICE + x{01} SDBEGINSQ + IF:<{ + DROP + 139 PUSHINT + }>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ + 16 PUSHPOW2DEC + THROWANY + }> + 140 PUSHINT + }> + 139 PUSHINT + EQUAL + }> +""" + + */ diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk new file mode 100644 index 000000000..60e9c2b5c --- /dev/null +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -0,0 +1,181 @@ +// encoded as TL/B `len: (## 8) data: (bits (len*8))` +type TelegramString = slice; + +fun TelegramString.packToBuilder(self, mutate b: builder) { + val bytes = self.remainingBitsCount() / 8; + b.storeUint(bytes, 8); + b.storeSlice(self); +} + +fun TelegramString.unpackFromSlice(mutate s: slice) { + val bytes = s.loadUint(8); + return s.loadBits(bytes * 8); +} + +type Custom8 = int; + +fun Custom8.packToBuilder(self, mutate b: builder) { + b.storeUint(self, 8) +} + +fun Custom8.unpackFromSlice(mutate s: slice) { + return s.loadUint(8) +} + +struct StorWithStr { + a: int32; + str: TelegramString; + b: int32; +} + +struct PointWithCustomInt { + a: Custom8; + b: int8; +} + + +type MyBorderedInt = int; + +fun MyBorderedInt.packToBuilder(self, mutate b: builder) { + if (self > 10) { b.storeUint(1, 4) } + else if (self > 0) { b.storeUint(2, 4) } + else { b.storeUint(3, 4) } +} + +fun MyBorderedInt.unpackFromSlice(mutate s: slice) { + return match (s.loadUint(4)) { + 1 => 10, + 2 => 0, + 3 => -1, + else => throw 123 + } +} + +struct WithMyBorder { + a: int8; + b: MyBorderedInt; +} + +type MyCustomNothing = (); + +struct WithFakeWriter { + a: int8; + fake: MyCustomNothing = (); + b: int8; +} + +fun MyCustomNothing.packToBuilder(self, mutate b: builder) { + b.storeUint(123, 32); + b.storeRef(createEmptyCell()); +} + +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + +type Tensor3Skipping1 = (int, int, int); + +fun Tensor3Skipping1.unpackFromSlice(mutate s: slice): Tensor3Skipping1 { + val e0 = s.loadUint(8); + val e2 = s.loadUint(8); + return (e0, 0, e2); +} + +fun Tensor3Skipping1.packToBuilder(self, mutate b: builder) { + b.storeUint(self.0, 8).storeUint(self.2, 8); +} + + +@method_id(101) +fun test1() { + var t: StorWithStr = { a: 10, str: "abc", b: 20 }; + var c = t.toCell(); + var back = c.load(); + return ((t.b == back.b) & (t.str.bitsEqual(back.str)), back.str.remainingBitsCount(), c.hash() & 0xFFFF); +} + +@method_id(102) +fun test2() { + var s = ("" as TelegramString, "" as TelegramString); + return beginCell().storeAny(s).endCell().beginParse().remainingBitsCount(); +} + +@method_id(103) +fun test3() { + return PointWithCustomInt.fromSlice(stringHexToSlice("0102")); +} + +@method_id(104) +fun test4(initialInt: int) { + var c = WithMyBorder { a: 0, b: initialInt }.toCell(); + return c.load().b; +} + +@method_id(105) +fun test5() { + var f: WithFakeWriter = { a: 10, b: 20 }; + var s = beginCell().storeAny(f).endCell().beginParse(); + val size = s.remainingBitsAndRefsCount(); + return (s.skipBits(8).loadUint(32), size.1); +} + +@method_id(106) +fun test6() { + gModByCustom = 6; + val m1: WithGlobalModifier = { a: 8, n: 16 }; + val c1 = m1.toCell(); + val r2 = WithGlobalModifier.fromSlice(stringHexToSlice("01FF02")); + val gAfter2 = gModByCustom; + WithGlobalModifier.fromSlice(stringHexToSlice("010002")); // not deleted, sets 0 + val gAfter3 = gModByCustom; + var s = stringHexToSlice("010902FF"); + s.skipAny(); // custom loader also called + return (c1.beginParse().skipBits(8).loadUint(8), r2.n, gAfter2, gAfter3, s.remainingBitsCount(), gModByCustom); +} + +@method_id(107) +fun test7() { + val t = (1, 2, 3) as Tensor3Skipping1; + __expect_type(t.toCell(), "Cell"); + return t.toCell().load(); +} + +fun main() { + __expect_type("" as TelegramString, "TelegramString"); +} + +/** +@testcase | 101 | | -1 24 5203 +@testcase | 102 | | 16 +@testcase | 103 | | 1 2 +@testcase | 104 | 55 | 10 +@testcase | 104 | 8 | 0 +@testcase | 104 | -5 | -1 +@testcase | 105 | | 123 1 +@testcase | 106 | | 6 2 255 0 8 9 +@testcase | 107 | | 1 0 3 + +@fif_codegen +""" + test3() PROC:<{ + x{0102} PUSHSLICE + 8 LDU + 8 PLDI + }> +""" + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 385aa3b53..0079871a6 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -1,113 +1,177 @@ -const cc1 = "0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"a; -const cc2 = "EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"a; +const cc1 = address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); +const cc2 = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); -fun verifyAddr(addr: slice, workchain: int, number: int) { - assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; - addr.skipBits(3); - assert (addr.loadUint(8) == workchain) throw 111; - assert (addr.loadUint(256) == number) throw 111; +fun verifyAddr(addr: address, workchain: int, number: int) { + assert (addr == addr) throw 111; + assert (!(addr != addr)) throw 111; + assert (addr.isInternal(), 111); + assert (!addr.isExternal(), 111); + assert (!addr.isNone(), 111); + assert (addr.bitsEqual(addr), 111); + + var (wc, h) = addr.getWorkchainAndHash(); + assert (wc == workchain) throw 111; + assert (h == number) throw 111; + assert (addr.getWorkchain() == workchain) throw 111; + + var s = addr as slice; + assert (s.remainingBitsCount() == 3 + 8 + 256) throw 112; + s.skipBits(3); + assert (s.loadInt(8) == workchain) throw 111; + assert (s.loadUint(256) == number) throw 111; +} + +fun address.createNone() { + return createAddressNone() +} + +@noinline +fun check0(addr: address) { + assert (addr.getWorkchain() == 0) throw 111; } +fun codegenAddrEq(a: address, b: address) { + if (a == b) { return 1; } + if (a != b) { return 2; } + return 3; +} + + fun main() { - verifyAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a, 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"a, 0, 0); - verifyAddr("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"a, 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); - verifyAddr("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"a, 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); - verifyAddr("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"a, 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); - verifyAddr("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"a, 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); - verifyAddr("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"a, 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); - verifyAddr("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"a, 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); - verifyAddr("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"a, 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); - verifyAddr("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"a, 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); - verifyAddr("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"a, 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); - verifyAddr("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"a, 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); - verifyAddr("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"a, 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); - verifyAddr("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"a, 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); - verifyAddr("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"a, 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); - verifyAddr("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"a, 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); - verifyAddr("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"a, 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); - verifyAddr("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"a, 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); - verifyAddr("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"a, 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); - verifyAddr("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"a, 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); - verifyAddr("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"a, 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); - verifyAddr("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"a, 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); - verifyAddr("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"a, 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); - verifyAddr("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"a, 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); - verifyAddr("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"a, 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); - verifyAddr("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"a, 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); - verifyAddr("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"a, 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); - verifyAddr("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"a, 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); - verifyAddr("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"a, 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); - verifyAddr("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"a, 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); - verifyAddr("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"a, 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); - verifyAddr("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"a, 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); - verifyAddr("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"a, 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); - verifyAddr("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"a, 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); - verifyAddr("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"a, 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); - verifyAddr("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"a, 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); - verifyAddr("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"a, 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); - verifyAddr("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"a, 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); - verifyAddr("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"a, 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); - verifyAddr("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"a, 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); - verifyAddr("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"a, 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); - verifyAddr("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"a, 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); - verifyAddr("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"a, 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); - verifyAddr("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"a, 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); - verifyAddr("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"a, 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); - verifyAddr("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"a, 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); - verifyAddr("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"a, 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); - verifyAddr("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"a, 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); - verifyAddr("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"a, 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); - verifyAddr("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"a, 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); - verifyAddr("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"a, 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); - verifyAddr("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"a, 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); - verifyAddr("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"a, 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); - verifyAddr("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"a, 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); - verifyAddr("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"a, 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); - verifyAddr("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"a, 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); - verifyAddr("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"a, 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); - verifyAddr("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"a, 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); - verifyAddr("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"a, 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); - verifyAddr("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"a, 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); - verifyAddr("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"a, 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); - verifyAddr("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"a, 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); - verifyAddr("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"a, 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); - verifyAddr("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"a, 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); - verifyAddr("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"a, 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); - verifyAddr("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"a, 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); - verifyAddr("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"a, 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); - verifyAddr("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"a, 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); - verifyAddr("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"a, 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"a, 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"a, 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); - verifyAddr("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"a, 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); - verifyAddr("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"a, 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); - verifyAddr("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"a, 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); - verifyAddr("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"a, 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); - verifyAddr("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"a, 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); - verifyAddr("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"a, 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"a, 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); - verifyAddr("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"a, 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"a, 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); - verifyAddr("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"a, 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); - verifyAddr("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"a, 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + __expect_type(cc1, "address"); - return cc1.isSliceBitsEqual(cc2); + verifyAddr(address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), -1, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); + verifyAddr(address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr(address("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr(address("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr(address("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), -1, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr(address("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), -1, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr(address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), -1, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr(address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), -1, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr(address("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), -1, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr(address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), -1, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), -1, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr(address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), -1, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr(address("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), -1, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), -1, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr(address("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), -1, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr(address("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), -1, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr(address("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), -1, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr(address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr(address("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr(address("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr(address("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr(address("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr(address("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr(address("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr(address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr(address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr(address("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr(address("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr(address("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr(address("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr(address("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr(address("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr(address("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr(address("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr(address("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr(address("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr(address("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr(address("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr(address("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr(address("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr(address("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr(address("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr(address("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr(address("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr(address("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr(address("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr(address("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr(address("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr(address("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr(address("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr(address("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr(address("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr(address("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr(address("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr(address("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr(address("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr(address("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr(address("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr(address("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr(address("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr(address("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr(address("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr(address("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr(address("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr(address("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr(address("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr(address("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr(address("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr(address("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr(address("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr(address("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr(address("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr(address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), -1, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr(address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), -1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + + return ( + cc1 == cc2, + cc2 != cc1, + (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), + createAddressNone() == cc1, + createAddressNone() == address.createNone(), + check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + ); } /** -@testcase | 0 | | -1 +@testcase | 0 | | -1 0 -1 0 -1 + +@fif_codegen +""" + check0() PROC:<{ // addr + REWRITESTDADDR + DROP // '2 + 0 EQINT // '4 + 111 THROWIFNOT // + }> +""" + +@fif_codegen +""" + codegenAddrEq() PROC:<{ // a b + 2DUP // a b a b + SDEQ // a b '2 + IFJMP:<{ // a b + 2DROP // + 1 PUSHINT // '3=1 + }> // a b + SDEQ // '4 + IFNOTJMP:<{ // + 2 PUSHINT // '5=2 + }> // + 3 PUSHINT // '6=3 + }> +""" */ diff --git a/tolk-tester/tests/pure-functions.tolk b/tolk-tester/tests/pure-functions.tolk index bb1a7d593..285320dec 100644 --- a/tolk-tester/tests/pure-functions.tolk +++ b/tolk-tester/tests/pure-functions.tolk @@ -11,7 +11,7 @@ fun f_pure2(): int { @pure fun get_contract_data(): (int, int) { - var c: cell = getContractData(); + var c: cell = contract.getData(); var cs: slice = c.beginParse(); cs.loadBits(32); var value: int = cs.loadUint(16); @@ -20,7 +20,7 @@ fun get_contract_data(): (int, int) { fun save_contract_data(value: int) { var b: builder = beginCell().storeInt(1, 32).storeUint(value, 16); - setContractData(b.endCell()); + contract.setData(b.endCell()); } @pure diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index 8e748ecf1..a6b910009 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -15,6 +15,7 @@ global used_gv: int; fun receiveGetter(): () -> int { return used_as_noncall2; } @pure +@noinline fun usedButOptimizedOut(x: int): int { return x + 2; } fun main(): (int, int, int) { @@ -31,18 +32,18 @@ fun main(): (int, int, int) { @testcase | 0 | | 3 10 20 -@fif_codegen DECLPROC used_as_noncall1 -@fif_codegen DECLGLOBVAR used_gv +@fif_codegen DECLPROC used_as_noncall1() +@fif_codegen DECLGLOBVAR $used_gv -@fif_codegen_avoid DECLPROC unused1 -@fif_codegen_avoid DECLPROC unused2 -@fif_codegen_avoid DECLPROC unused3 -@fif_codegen_avoid DECLGLOBVAR unused_gv +@fif_codegen_avoid DECLPROC unused1() +@fif_codegen_avoid DECLPROC unused2() +@fif_codegen_avoid DECLPROC unused3() +@fif_codegen_avoid DECLGLOBVAR $unused_gv Note, that `usedButOptimizedOut()` (a pure function which result is unused) is currently codegenerated, since it's formally reachable. This is because optimizing code is a moment of codegen for now (later than marking unused symbols). -@fif_codegen DECLPROC usedButOptimizedOut -@fif_codegen_avoid usedButOptimizedOut CALLDICT +@fif_codegen DECLPROC usedButOptimizedOut() +@fif_codegen_avoid usedButOptimizedOut() CALLDICT */ diff --git a/tolk-tester/tests/s1.tolk b/tolk-tester/tests/s1.tolk deleted file mode 100644 index c7c4f6946..000000000 --- a/tolk-tester/tests/s1.tolk +++ /dev/null @@ -1,61 +0,0 @@ -get ascii_slice(): slice { - return"string"; -} - -get raw_slice(): slice { - return "abcdef"s; -} - -get addr_slice(): slice { - return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; -} - -get string_hex(): int { - return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; -} - -get fun string_minihash(): int { // 'get' and 'get fun' both possible - return "transfer(slice, int)"h; -} - -get fun string_maxihash(): int { - return "transfer(slice, int)"H; -} - -get fun string_crc32(): int { - return "transfer(slice, int)"c; -} - -@pure -fun newc(): builder -asm "NEWC"; -fun endcs(b: builder): slice -asm "ENDC" "CTOS"; -@pure -fun sdeq(s1: slice, s2: slice): int -asm "SDEQ"; - -fun main() { - var s_ascii: slice = ascii_slice(); - var s_raw: slice = raw_slice(); - var s_addr: slice = addr_slice(); - var i_hex: int = string_hex(); - var i_mini: int = string_minihash(); - var i_maxi: int = string_maxihash(); - var i_crc: int = string_crc32(); - assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; - assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; - assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) - .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); - assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; - assert(i_mini == 0x7a62e8a8) throw 105; - assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; - assert(i_crc == 2235694568) throw 107; - return 0; -} - -/** -@testcase | 0 | | 0 - -@code_hash 13830542019509784148027107880226447201604257839069192762244575629978154217223 -*/ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index b0567696d..5fa5538dd 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -1,18 +1,22 @@ -fun incChained(mutate self: int): self { +@noinline +fun int.incChained(mutate self): self { self = self + 1; return self; } -fun incChained2(mutate self: int): self { +@noinline +fun int.incChained2(mutate self): self { return self.incChained(); } -fun incChained3(mutate self: int): self { - incChained(mutate self); +@noinline +fun int.incChained3(mutate self): self { + self.incChained(); return self; } -fun incChained4(mutate self: int): self { +@noinline +fun int.incChained4(mutate self): self { self.incChained(); return self; } @@ -25,7 +29,7 @@ fun testIncChainedCodegen(x: int) { @method_id(102) fun testIncChained() { var x: int = 10; - incChained(mutate x); + x.incChained(); x.incChained(); x.incChained2(); x.incChained2().incChained(); @@ -34,7 +38,7 @@ fun testIncChained() { return x.incChained(); } -fun incChainedWithMiddleReturn(mutate self: int, maxValue: int): self { +fun int.incChainedWithMiddleReturn(mutate self, maxValue: int): self { if (self >= maxValue) { return self; } @@ -49,7 +53,7 @@ fun testIncChainedWithMiddleReturn(x: int) { return x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(999); } -fun incChainedMutatingBoth(mutate self: int, mutate y: int): self { +fun int.incChainedMutatingBoth(mutate self, mutate y: int): self { self += 1; y += 1; return self; @@ -62,12 +66,12 @@ fun testIncChainedMutatingBoth() { var (x, y) = (0, 0); c104 = 0; x.incChainedMutatingBoth(mutate y).incChainedMutatingBoth(mutate y); - incChainedMutatingBoth(mutate x, mutate y); + x.incChainedMutatingBoth(mutate y); x = x.incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate y); return (x, y, c104); } -fun incTensorChained(mutate self: (int, int)): self { +fun (int, int).incTensorChained(mutate self): self { val (f, s) = self; self = (f + 1, s + 1); return self; @@ -80,7 +84,7 @@ fun testIncTensorChained(f: int, s: int) { return tens.incTensorChained().incTensorChained(); } -fun incConditionalChainable(mutate self: int, mutate another: int, ifLessThan: int): self { +fun int.incConditionalChainable(mutate self, mutate another: int, ifLessThan: int): self { another += 1; return self.incChained() < ifLessThan ? self.incChained().incChained() : self; } @@ -93,7 +97,7 @@ fun testIncConditionalChainable(x: int) { return (x.incConditionalChainable(mutate y, 5), y); } -fun checkNotEq(self: int, throwIfEq: int): void { +fun int.checkNotEq(self, throwIfEq: int): void { if (self == throwIfEq) { throw 100 + throwIfEq; } @@ -113,7 +117,7 @@ fun testNotMutatingSelf(arg: int) { global c108: int; -fun checkNotEqChainable(self: int, throwIfEq: int): self { +fun int.checkNotEqChainable(self, throwIfEq: int): self { c108 += 1; if (self != throwIfEq) { return self; @@ -136,7 +140,7 @@ fun testNotMutatingChainableSelf(arg: int) { global onceFailed109: int; -fun checkNotEqChainableMutateAnother(self: int, throwIfEq: int, mutate toInc: int): self { +fun int.checkNotEqChainableMutateAnother(self, throwIfEq: int, mutate toInc: int): self { if (onceFailed109) { return self; } toInc += 1; try { return self.checkNotEqChainable(throwIfEq); } @@ -158,9 +162,9 @@ fun testNotMutatingChainableSelfMutateAnother(initial: int) { return (arg, c108, c109, x); } -fun pickG110(mutate self: int, mutate pushTo: tuple): self { +fun int.pickG110(mutate self, mutate pushTo: tuple): self { self += 10; - pushTo.tuplePush(c110); + pushTo.push(c110); return self; } @@ -175,13 +179,13 @@ fun testMutateGlobalsLValue(init: int) { return (c110, tup110); } -fun myTuplePush(mutate self: tuple, value: T): self { - self.tuplePush(value); +fun tuple.myTuplePush(mutate self, value: T): self { + self.push(value); return self; } -fun myTupleAt(self: tuple, idx: int): T { - return self.tupleAt(idx); +fun tuple.myTupleAt(self, idx: int): T { + return self.get(idx); } global tup111: tuple; @@ -192,7 +196,7 @@ fun testForallFunctionsWithSelf(): (int, int, tuple) { tup111 = createEmptyTuple(); t.myTuplePush(10); tup111.myTuplePush(1).myTuplePush(2).myTuplePush(3); - return (t.myTupleAt(0), tup111.myTupleAt(tup111.tupleSize() - 1), tup111); + return (t.myTupleAt(0), tup111.myTupleAt(tup111.size() - 1), tup111); } @@ -222,32 +226,29 @@ fun main() { } @fif_codegen """ - incChained PROC:<{ - // self - INC // self + int.incChained() PROC:<{ // self + INC // self }> - incChained2 PROC:<{ - // self - incChained CALLDICT // self + int.incChained2() PROC:<{ // self + int.incChained() CALLDICT // self }> - incChained3 PROC:<{ - // self - incChained CALLDICT // self + int.incChained3() PROC:<{ // self + int.incChained() CALLDICT // self }> - incChained4 PROC:<{ - // self - incChained CALLDICT // self + int.incChained4() PROC:<{ // self + int.incChained() CALLDICT // self }> """ @fif_codegen """ - testIncChainedCodegen PROC:<{ - // x - incChained CALLDICT // x - incChained2 CALLDICT // x - incChained3 CALLDICT // x - incChained4 CALLDICT // x + testIncChainedCodegen() PROC:<{ // x + int.incChained() CALLDICT // x + int.incChained2() CALLDICT // x + int.incChained3() CALLDICT // x + int.incChained4() CALLDICT // x }> """ + +@fif_codegen int.checkNotEqChainableMutateAnother() PREPAREDICT */ diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk new file mode 100644 index 000000000..3a889dcf4 --- /dev/null +++ b/tolk-tester/tests/send-msg-1.tolk @@ -0,0 +1,755 @@ +import "@stdlib/tvm-dicts" +import "@stdlib/tvm-lowlevel" + +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline_ref +fun calculateNftItemStateInitData(itemIndex: int): cell { + return beginCell() + .storeUint(itemIndex, 64) + .storeAddress(getMyAddressDev()) + .endCell(); +} + +@inline +fun calculateStateInitCell(code: cell, data: cell): cell { + return beginCell() + .storeUint(0, 2) // 0 split_depth, 0 special + .storeDict(code) + .storeDict(data) + .storeUint(0, 1) // 0 library + .endCell(); +} + +fun calculateNftItemAddress(workchain: int, stateInitCell: cell): address { + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + 0 split_depth + .storeInt(workchain, 8) + .storeUint(stateInitCell.hash(), 256) + .endCell() + .beginParse() as address; +} + +struct(0x12345678) MyBody { + queryId: uint64; +} + +@noinline +fun test1_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1() { + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body: body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test1_manual().hash(), 101); + return b.hash(); +} + +@noinline +fun test2_manual() { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); +} + +@method_id(102) +fun test2() { + val body: MyBody = { queryId: 127493264572 }; + var b = createMessage({ + body, + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test2_manual().hash(), 102); + return b.hash(); +} + +@noinline +fun test3_manual() { + val body_ref = beginCell() + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(body_ref) + .endCell(); +} + +@method_id(103) +fun test3() { + var b = createMessage({ + body: MyBody{ queryId: 127493264572 }.toCell(), + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test3_manual().hash(), 103); + return b.hash(); +} + +@noinline +fun test4_manual(bodyCell: cell, dest: address, value: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(dest) + .storeCoins(value) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyCell) + .endCell(); +} + +@method_id(104) +fun test4(value: coins) { + val bodyCell = beginCell() + .storeUint(0x03738FA9, 32) + .storeBool(true).storeBool(false).storeCoins(123) + .endCell(); + val dest = address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"); + var b = createMessage({ + body: bodyCell, + value, + bounce: true, + dest: dest, + }); + assert(b.hash() == test4_manual(bodyCell, dest, value).hash(), 104); + return b.hash(); +} + +@noinline +fun test5_manual() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(105) +fun test5() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (123, ec_dict), + }); + assert(b.hash() == test5_manual().hash(), 105); + return b.hash(); +} + +@noinline +fun test6_manual(value: coins, ec_dict: dict) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(value) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(106) +fun test6(value: coins, dictKey: int?) { + var ec_dict = createEmptyDict(); + if (dictKey != null) { + ec_dict.iDictSet(32, dictKey, "ec1"); + } + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (value, ec_dict), + body, + }); + assert(b.hash() == test6_manual(value, ec_dict).hash(), 106); + return b.hash(); +} + +struct MyNftBody { + nftContent: cell; +} + +@noinline +fun test7_manual(nftItemCode: cell, amount: coins) { + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val nftItemData = calculateNftItemStateInitData(itemIndex); + val nftAddress = calculateNftItemAddress(BASECHAIN, calculateStateInitCell(nftItemCode, nftItemData)); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(nftItemCode) + .storeRef(nftItemData) + .storeRef(nftContent) + .endCell(); +} + +@method_id(107) +fun test7(amount: coins) { + val nftItemCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val stateInitData = calculateNftItemStateInitData(itemIndex); + + val body: MyNftBody = { nftContent }; + var b = createMessage({ + bounce: true, + body, + dest: { workchain: BASECHAIN, stateInit: { code: nftItemCode, data: stateInitData } }, + value: amount, + }); + assert(b.hash() == test7_manual(nftItemCode, amount).hash(), 107); + return b.hash(); +} + +struct(0x706c7567) RequestPaymentMessage { + queryId: uint64; + amount: coins; + someDict: dict; +} + +@noinline +fun test8_manual(destAddr: address, requestedAmount: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(destAddr) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x706c7567, 32) + .storeUint(0, 64) + .storeCoins(requestedAmount) + .storeDict(null) + .endCell(); +} + +@method_id(108) +fun test8(requestedAmount: coins) { + val destAddr = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + var b = createMessage({ + bounce: true, + dest: destAddr, + value: 123, + body: RequestPaymentMessage { + queryId: 0, + amount: requestedAmount, + someDict: null, + } + }); + assert(b.hash() == test8_manual(destAddr, requestedAmount).hash(), 108); + return b.hash(); +} + +@noinline +fun test9_manual(bounceable: bool) { + return beginCell() + .storeUint(0b01, 2).storeBool(bounceable).storeUint(0b000, 3) + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(0) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .storeInt(-777, 13) + .endCell(); +} + +@method_id(109) +fun test9(bounceable: bool) { + var b = createMessage({ + bounce: bounceable, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 0, + body: (0x12345678 as uint32, 800 as uint64, -777 as int13), + }); + assert(b.hash() == test9_manual(bounceable).hash(), 109); + return b.hash(); +} + +type Body500Bits = (uint250, uint250); // this body guaranteely fits into cell + +@noinline +fun test10_manual(bd: Body500Bits) { + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 118) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd.0, 250) + .storeUint(bd.1, 250) + .endCell(); +} + +@method_id(110) +fun test10(bd: Body500Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 118, + }); + assert(b.hash() == test10_manual(bd).hash(), 110); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +@noinline +fun test11_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 110) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(111) +fun test11(bd: Body750Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 110, + }); + assert(b.hash() == test11_manual(bd).hash(), 111); + return b.hash(); +} + +@noinline +fun test12_manual(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeUint(data32, 32) + .endCell(); +} + +@method_id(112) +fun test12(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + var b = createMessage({ + bounce: true, + body: data32, + dest: { workchain: MASTERCHAIN, stateInit: init }, + value: ton("0.05"), + }); + assert(b.hash() == test12_manual(data32).hash(), 112); + return b.hash(); +} + +@noinline +fun test13_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K")) + .storeCoins(52 << 78) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .endCell(); // no body at all +} + +@method_id(113) +fun test13() { + var b = createMessage({ + bounce: true, + dest: address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), + value: 52 << 78, + }); + assert(b.hash() == test13_manual().hash(), 113); + return b.hash(); +} + +@noinline +fun test14_manual(stateInitCell: cell, amount: coins) { + val nftAddress = calculateNftItemAddress(BASECHAIN, stateInitCell); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 1 either right (state init ref), 0 either left (body inline) + .storeUint(0b110, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .endCell(); // no body +} + +@method_id(114) +fun test14(amount: coins) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = calculateStateInitCell(stateInitCode, stateInitData); + + var b = createMessage({ + bounce: true, + dest: { stateInit: stateInitCell }, + value: amount, + }); + assert(b.hash() == test14_manual(stateInitCell, amount).hash(), 114); + return b.hash(); +} + +struct(0x1234) Body15 { + tens: Body750Bits; + more: int32; +} + +@noinline +fun test15_manual(stateInitCell: cell, bd: Body15) { + val bodyRef = beginCell().storeUint(0x1234,16).storeUint(bd.tens.0,250).storeUint(bd.tens.1,250).storeUint(bd.tens.2,250).storeInt(bd.more,32).endCell(); + val nftAddress = calculateNftItemAddress(MASTERCHAIN, stateInitCell); + + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(nftAddress) + .storeCoins(ton("100.0004")) + // 1 state init exists, 1 either right (state init ref), 1 either right (body ref) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .storeRef(bodyRef) + .endCell(); // no body +} + +@method_id(115) +fun test15(tens0: int, tens1: int) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = beginCell() + .storeUint(1, 1) // has split_depth + .storeUint(3, 5) // split_depth + .storeUint(0, 1) // 0 special + .storeDict(stateInitCode) + .storeDict(stateInitData) + .storeUint(0, 1) // 0 library + .endCell(); + var bd: Body15 = { + tens: (1 << 88, tens0, tens1), + more: max(tens0, tens1), + }; + + var b = createMessage({ + bounce: false, + dest: { workchain: MASTERCHAIN, stateInit: stateInitCell }, + body: bd, + value: ton("100.0004"), + }); + assert(b.hash() == test15_manual(stateInitCell, bd).hash(), 115); + return b.hash(); +} + +@noinline +fun test18_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeBuilder(dest) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(118) +fun test18(queryId: uint64) { + val body: MyBody = { queryId }; + var b = createMessage({ + body: body, + bounce: true, + dest: beginCell().storeUint(0, 2), + value: 123, + }); + assert(b.hash() == test18_manual(beginCell().storeUint(0, 2), queryId).hash(), 118); + return b.hash(); +} + +@noinline +fun test19_manual(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeBuilder(destB) + .storeCoins(ton("0.059")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd, 64) + .endCell(); +} + +@method_id(119) +fun test19(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + var b = createMessage({ + bounce: 10 < 3, + body: bd, + dest: destB, + value: ton("0.059"), + }); + assert(b.hash() == test19_manual(bd).hash(), 119); + return b.hash(); +} + +@noinline +fun test20_manual() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyB.endCell()) + .endCell(); +} + +@method_id(120) +fun test20() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createMessage({ + body: bodyB, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test20_manual().hash(), 120); + return b.hash(); +} + +struct(0x12345678) Body21 { + queryId: uint64; + payload: RemainingBitsAndRefs; +} + +@noinline +fun test21_manual(bd: Body21) { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi")) + .storeCoins(ton("0.1")) // value.grams + .storeDict(createEmptyDict()) + .storeUint(1, 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bd.toCell()) // stored as ref due to remainder of unpredictable size + .endCell(); +} + +@method_id(121) +fun test21(queryId: uint64) { + var payloadSlice = beginCell().storeUint(0x12345678, 32).storeUint(800, 64).storeRef(createEmptyCell()).endCell().beginParse(); + var body: Body21 = { queryId, payload: payloadSlice }; + var b = createMessage({ + bounce: false, + dest: address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), + value: (ton("0.1"), createEmptyDict()), + body, + }); + assert(b.hash() == test21_manual(body).hash(), 121); + return b.hash(); +} + +@noinline +fun test22_manual(unsafeB: builder) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + .storeCoins(ton("0.08")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (no ref) + .storeBuilder(unsafeB) + .endCell(); +} + +@method_id(122) +fun test22(op: uint32, queryId: uint64) { + var bodyB = beginCell().storeUint(op, 32).storeUint(queryId, 64); + var b = createMessage({ + bounce: true, + dest: address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), + value: ton("0.08"), + body: UnsafeBodyNoRef { + forceInline: bodyB + }, + }); + assert(b.hash() == test22_manual(bodyB).hash(), 122); + return b.hash(); +} + +struct Body23Unlimited { + any1: (address, builder), + any2: RemainingBitsAndRefs; +} + +@noinline +fun test23_manual(state32: uint32, body: Body23Unlimited) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(9, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.10009")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeAddress(body.any1.0) + .storeBuilder(body.any1.1) + .storeSlice(body.any2) + .endCell(); +} + +@method_id(123) +fun test23(state32: uint32) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + var bd: Body23Unlimited = { + any1: (createAddressNone(), beginCell().storeRef(init.code)), + any2: beginCell().storeUint(80, 80).endCell().beginParse(), + }; + + var b = createMessage({ + bounce: true, + body: UnsafeBodyNoRef { + forceInline: bd, + }, + dest: { workchain: 9, stateInit: init }, + value: ton("0.10009"), + }); + assert(b.hash() == test23_manual(state32, bd).hash(), 123); + return b.hash(); +} + +@noinline +fun test24_manual(addrHash: uint256) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(addrHash, 256) + .storeCoins(ton("0.6")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(124) +fun test24() { + val addrHash = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA").getWorkchainAndHash().1; + var b = createMessage({ + body: MyBody { queryId: 800 }, + bounce: true, + dest: (MASTERCHAIN, addrHash), + value: ton("0.6"), + }); + assert(b.hash() == test24_manual(addrHash).hash(), 124); + return b.hash(); +} + +fun main() { +} + +/** +@testcase | 101 | | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 102 | | 20192939504955637835708665496574868659039935190188156593169026529135727309085 +@testcase | 103 | | 97654287980681401436727082042373998183264988661375688119718809500301413968039 +@testcase | 104 | 600000 | 755636693689039391990782995030008885663781699175339033402019272057515711062 +@testcase | 105 | | 24816341499673835567887890014278904436471582322802948121781418622643959482495 +@testcase | 106 | 123 null | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 106 | 456 8888 | 39337586036945718311402746340438400160817844833530971545330721291986281100430 +@testcase | 107 | 1000 | 55093441331748625324828489600632232039914212774002148634088483962817636598198 +@testcase | 108 | 50000000 | 95023796475113775225029817428715936488418545169963429399979521091689824066088 +@testcase | 109 | 0 | 55999621586681214992294941423256376619779969729861696464321825639854258502733 +@testcase | 109 | -1 | 84087871798432599249441213206223143701565541307347047545146076475041341315422 +@testcase | 110 | 250 250 | 97468400996544929599099493087921300963923138788231489050737873840992619823773 +@testcase | 111 | 1 2 3 | 35151166799433266221446406287469019610025742512503320058804207122452431754243 +@testcase | 112 | 32 | 40256061350602595831367445407067573081836468277788226383346273736379122699330 +@testcase | 113 | | 8212062468046185391185852622213582155366804215840270337189205672457136520017 +@testcase | 114 | 100500 | 69207815800109735757177433421533576767812185821226447066044060358661780329301 +@testcase | 115 | 66 77 | 77952119695754644819736002369288963466111166174255607433142955344110346202253 +@testcase | 118 | 888 | 75462675913935779917745192355822465171309245151518256862408373999119088535160 +@testcase | 119 | 999 | 20451206881650273118327988889219529836875916996856547550069532964562229905067 +@testcase | 120 | | 86341416901030824925289599533989709413619468614523233983159539599569269903295 +@testcase | 121 | 2983742 | 29682465216061902145511914581895871811826194319877849112468723187093436476183 +@testcase | 122 | 88 12892 | 104095783372529353117379287284649790470404208891518326338200063605256169961669 +@testcase | 123 | 999 | 93691386126134034953606952841435771108648354412078750046821446829279390964584 +@testcase | 124 | | 21886688052816798288463190773103865772534937765373272039597499398551023701577 + */ diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk new file mode 100644 index 000000000..050cd8432 --- /dev/null +++ b/tolk-tester/tests/send-msg-2.tolk @@ -0,0 +1,340 @@ +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline +fun getAddressShard(address: address, shardLen: int): int { + // Skip workchain, load shard prefix + (address as slice).skipBits(3 + 8); + return (address as slice).loadUint(shardLen); +} + +@inline +fun packJettonWalletData(status: int, balance: int, ownerAddress: address, jettonMasterAddress: address): cell { + return beginCell() + .storeUint(status, 8) + .storeCoins(balance) + .storeAddress(ownerAddress) + .storeAddress(jettonMasterAddress) + .endCell(); +} + +@inline +fun calculateJettonWalletStateInitWithShard(ownerAddress: address, jettonMasterAddress: address, jettonWalletCode: cell): cell { + /* + https://github.com/ton-blockchain/ton/blob/8a9ff339927b22b72819c5125428b70c406da631/crypto/block/block.tlb#L144 + _ split_depth:(Maybe (## 5)) special:(Maybe TickTock) + code:(Maybe ^Cell) data:(Maybe ^Cell) + library:(Maybe ^Cell) = StateInit; + */ + return beginCell() + .storeUint(1, 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0, 1) + .storeMaybeRef(jettonWalletCode) + .storeMaybeRef( + packJettonWalletData( + 0, // status + 0, // balance + ownerAddress, + jettonMasterAddress) + ) + .storeUint(0, 1) // Empty libraries + .endCell(); +} + +@inline +fun calculateJettonWalletAddress(shardPrefix: int, stateInitCell: cell): address { + var mask = (1 << (256 - SHARD_DEPTH)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, SHARD_DEPTH) + .storeUint(prefixLess, 256 - SHARD_DEPTH) + .endCell() + .beginParse() as address; +} + +@inline +fun calculateAddressInAnotherShard(pivotAddress: address, shardPrefixLen: uint5, code: cell, data: cell): builder { + val stateInitCell = beginCell() + .storeUint(1, 1) + .storeUint(shardPrefixLen, 5) // shard depth + .storeUint(0, 1) + .storeMaybeRef(code) + .storeMaybeRef(data) + .storeUint(0, 1) // Empty libraries + .endCell(); + + val shardPrefix = getAddressShard(pivotAddress, shardPrefixLen); + var mask = (1 << (256 - shardPrefixLen)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, shardPrefixLen) + .storeUint(prefixLess, 256 - shardPrefixLen); +} + +@noinline +fun test1_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInitCell); + var jettonWalletData = packJettonWalletData(0, 0, toAddress, getMyAddressDev()); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 1 either left (fixed_prefix_length exists) + .storeUint(0b101, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0b01101, 4 + 1) // 0 ticktock + 1 code + 1 data + 0 library + 1 body ref + .storeRef(jettonWalletCode) + .storeRef(jettonWalletData) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(101) +fun test1() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().storeUint(1<<14, 32).endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: jettonWalletCode, + data: packJettonWalletData(0, 0, toAddress, getMyAddressDev()), + }, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test1_manual(toAddress, masterMsg, jettonWalletCode).hash(), 101); + return b.hash() +} + +@noinline +fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInit = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInit); + + return beginCell() + .storeUint(0x10, 6) // not bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) // 1 state init exists + 1 state init ref + 1 body ref + .storeRef(stateInit) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(102) +fun test2() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + + var b = createMessage({ + bounce: false, + dest: { + workchain: BASECHAIN, + stateInit: stateInitCell, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test2_manual(toAddress, masterMsg, jettonWalletCode).hash(), 102); + return b.hash() +} + +struct Test3Body { + bigData: bits800; +} + +@noinline +fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { + var addrInShard = calculateAddressInAnotherShard(getMyAddressDev(), 20, myCode, myData); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeBuilder(addrInShard) // dest + .storeCoins(ton("0.001")) + .storeUint(0b10, 1 + 4 + 4 + 64 + 32 + 1 + 1) // 1 state init exists + 1 state init inline + .storeUint(1, 1) + .storeUint(20, 5) // shard depth + .storeUint(0b0110, 4) // code + data exist + .storeUint(1, 1) // body ref + .storeRef(myCode) + .storeRef(myData) + .storeRef(msgBody.toCell()) + .endCell(); +} + +@method_id(103) +fun test3() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).storeMaybeRef(myCode).endCell(); + val builder800 = beginCell().storeInt(1, 250).storeInt(1, 250).storeInt(1, 250).storeInt(1, 50); + val body: Test3Body = { bigData: builder800.endCell().beginParse() as bits800 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 20, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.001"), + }); + assert(b.hash() == test3_manual(myCode, myData, body).hash(), 103); + return b.hash() +} + +struct Test4Body { + f1: int64; + f2: int64; +} + +@method_id(104) +fun test4() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test4Body = { f1: 123, f2: 456 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 30, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.1"), + }); + return b.hash() +} + +@overflow1023_policy("suppress") +struct Test5Body { + c1: coins; c2: coins; c3: coins; + c4: coins; c5: coins; c6: coins; + c7: coins; c8: coins; c9: coins; +} + +@method_id(105) +fun test5() { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 2, + closeTo: pivot, + } + }, + body: UnsafeBodyNoRef { forceInline: body }, + value: ton("0.1"), + }); + return b.hash() +} + +@method_id(106) +fun test6(shardLen: int) { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + workchain: BASECHAIN, + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: shardLen, + closeTo: pivot, + } + }, + body: body, + value: ton("0.1"), + }); + return b.hash() +} + +fun main() {} + + +/** +@testcase | 101 | | 114692697425779687591180352274328586758984823673144676907237399062035787598876 +@testcase | 102 | | 19141316219057792078125032524380927286847294424548966584483011017258286475706 +@testcase | 103 | | 33184074158306258754479533493449661031201063538320283251640171344508567641003 +@testcase | 104 | | 52060388210984960140400107426752119783720473036191582802152450095687953630545 +@testcase | 105 | | 18248582963085351579261363895401647616194364201057112283341027605548475235397 +@testcase | 106 | 2 | 73367500160949078368695742399322272058735198813159066885436981754975296689213 +@testcase | 106 | 9 | 26260034723231226694451786564335585031223880314377072827334911071856417531179 + */ diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk new file mode 100644 index 000000000..8db47589d --- /dev/null +++ b/tolk-tester/tests/send-msg-3.tolk @@ -0,0 +1,238 @@ +/* +ext_out_msg_info$11 + src:MsgAddress // 00 on send + dest:MsgAddressExt // parameter + created_lt:uint64 // 0 on send + created_at:uint32 // 0 on send + = CommonMsgInfoRelaxed; +message$_ {X:Type} + info:CommonMsgInfoRelaxed + init:(Maybe (Either StateInit ^StateInit)) // 0 on send + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; +*/ + +struct(0x12345678) MyBody { + queryId: uint64; +} + +@noinline +fun test1_manual(topic: int) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(topic, 256) // external_address:(bits len) (assume it fits 248 bits) + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1(topic: int) { + val body: MyBody = { queryId: 800 }; + var b = createExternalLogMessage({ + body: body, + dest: ExtOutLogBucket { topic }, + }); + assert(b.hash() == test1_manual(topic).hash(), 101); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +@noinline +fun test2_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(1 << 102, 256) // external_address:(bits len) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(102) +fun test2(bd: Body750Bits) { + var b = createExternalLogMessage({ + body: bd, + dest: { topic: 1 << 102 }, + }); + assert(b.hash() == test2_manual(bd).hash(), 102); + return b.hash(); +} + +struct Body3 { + bodyAddress: address; + amount: coins; + someRef: cell; +} + +@method_id(103) +fun test3(eventId: int) { + val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + val someRef = MyBody { queryId: 800 }.toCell(); + var b = createExternalLogMessage({ + body: Body3 { bodyAddress, someRef, amount: ton("0.05") }, + dest: { topic: eventId }, + }); + return b.hash(); +} + +@noinline +fun test4_manual(dest: address, body: cell) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeAddress(dest) // dest (manually created external address) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(body) + .endCell(); +} + +@method_id(104) +fun test4() { + val body: cell = Body3 { bodyAddress: createAddressNone(), amount: 0, someRef: createEmptyCell() }.toCell(); + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test4_manual(dest, body).hash(), 104); + return b.hash(); +} + +@method_id(105) +fun test5(topic: int) { + var b = createExternalLogMessage({ + dest: { topic }, + }); + return b.hash(); +} + +@method_id(106) +fun test6() { + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + dest, + }); + return b.hash(); +} + +@noinline +fun test7_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeBuilder(dest) // manually created bits representing external address + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(107) +fun test7(queryId: uint64) { + val body: MyBody = { queryId }; + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12); + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test7_manual(dest, queryId).hash(), 107); + return b.hash(); +} + +@method_id(108) +fun test8(topic: int) { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createExternalLogMessage({ + body: bodyB, + dest: { topic }, + }); + return b.hash(); +} + +struct Body9Unlimited { + any1: cell?, + any2: builder, +} + +@method_id(109) +fun test9(topic: int) { + var bd: Body9Unlimited = { + any1: null, + any2: beginCell().storeUint(80, 80).storeMaybeRef(null).storeRef(createEmptyCell()), + }; + var b = createExternalLogMessage({ + body: UnsafeBodyNoRef { + forceInline: bd, + }, + dest: ExtOutLogBucket { topic: topic }, + }); + return b.hash(); +} + +@noinline +fun test10_manual(topic: slice, queryIdRef: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(0x00, 8) // prefix of ExtOutLogBucket + .storeSlice(topic) // assume this slice is bits248 + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + .storeRef(beginCell().storeUint(queryIdRef, 64).endCell()) + .endCell(); +} + +struct Body10 { + queryIdRef: Cell; +} + +@method_id(110) +fun test10() { + val queryIdRef = 123; + val topic = stringHexToSlice("012345678901234567890123456789012345678901234567890123456789FF"); + var b = createExternalLogMessage({ + body: Body10 { queryIdRef: (queryIdRef as uint64).toCell() }, + dest: ExtOutLogBucket { topic: topic as bits248 }, + }); + assert(b.hash() == test10_manual(topic, queryIdRef).hash(), 110); + return b.hash(); +} + +@method_id(111) +fun test11() { + try { + createExternalLogMessage({ + dest: ExtOutLogBucket { topic: "asdf" as bits248 } // wrong bits count + }); + return 0; + } catch (excno) { + return excno; + } +} + +fun main() { + +} + +/** +@testcase | 101 | 100500 | 8399954633429803564068357060778961153860921657197909539205508789211284886940 +@testcase | 102 | 1 2 3 | 64371641017422166665232911624005896033238822098768040861805756890339950064655 +@testcase | 103 | 829999 | 33039908261421267188072304726227928987040300641614103065684678216731595546507 +@testcase | 104 | | 94681566274361007141065727833953679134419126606639224444485202401996571399154 +@testcase | 105 | 329889 | 24202801754361933063453004029355483858376498707977868951519201997233775688005 +@testcase | 106 | | 17838299862930869444234264325786950440464929864630237676886309244018701054708 +@testcase | 107 | 777 | 77493748753370371314291958955179535305777482888906702570602071501642726442389 +@testcase | 108 | 0 | 69287572703823759594480986943855745035158339431740068559880301501812484359455 +@testcase | 109 | 12 | 72694183679327534355706107347199645597776533707531061585024123910830113048519 +@testcase | 110 | | 33186381366677843958628736306577552541950719844529663083708985644217867888324 +@testcase | 111 | | 9 + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 4d71bb63a..90f1ae644 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -1,17 +1,37 @@ // the goal of this file is not only to @testcase results — // but to check that this file compiles +struct Point { + x: int; + y: int; +} + +struct PointOfNulls { + x: int?; + y: int?; + sub: Point?; +} + +fun rand(): int + asm "RANDU256"; + fun getNullableInt(): int? { return 5; } fun getNullableSlice(): slice? { return null; } fun takeNullableInt(a: int?) {} fun takeNullableSlice(a: slice?) {} -fun increment(mutate self: int) { self += 1; } -fun assignToInt(mutate self: int, value: int) { self = value; } -fun assignToNullableInt(mutate self: int?, value: int) { self = value; } +fun increment(mutate x: int) { x += 1; } +fun int.increment(mutate self) { self += 1; } +fun assignToInt(mutate x: int, value: int) { x = value; } +fun int.assignToInt(mutate self, value: int) { self = value; } +fun assignToNullableInt(mutate x: int?, value: int) { x = value; } +fun int?.assignToNullableInt(mutate self, value: int) { self = value; } fun sameTensor(t: (int, int)) { return t; } -fun sameTensor2(t: (int?, (slice, slice, slice, builder)?)) { return t; } +fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } fun getTwo(): X { return 2 as X; } +fun assignNullTo(mutate v: T?) { v = null; } + +type Pair4N = (slice, slice, slice, builder)?; fun test1(): int { var x = getNullableInt(); @@ -44,9 +64,9 @@ fun test3(): int { } return x; } - if (random() > -1) { - if (y == null) { return -1; } - else { return y; } + if (rand() > -1) { + if (y == null) { return -1 } + else { return y } } return 0; } @@ -74,7 +94,7 @@ fun test5() { var (a, (b, c)) = (getNullableInt(), (getNullableInt(), getNullableInt())); if (a == null) { return -1; } if (!(b != null)) { return -2; } - if (random() ? c == null && c == null : c == null) { return -3; } + if (rand() ? c == null && c == null : c == null) { return -3; } return a + b + c; } @@ -85,10 +105,10 @@ fun test6() { __expect_type(a == null ? "" : a, "int"); takeNullableInt(a); __expect_type(a, "int"); - if (random()) { + if (rand()) { a = null; } else { - if (random()) { a = null; } + if (rand()) { a = null; } else { a = null; } } __expect_type(a, "null"); @@ -107,7 +127,7 @@ fun test7() { if (a == null && true) { return -1; } if (true && true && 1 && !0 && b == null) { return -2; } if (true ? c == null && (((c))) == null && true : false) { return -3; } - if (!true ? random() > 0 : a != null && (d == null && b != null)) { return -4; } + if (!true ? rand() > 0 : a != null && (d == null && b != null)) { return -4; } return a + b + c + d; } @@ -115,7 +135,7 @@ fun test8(x: int?, y: int?) { var allGt1 = x != null && x > 1 && y != null && y > 1; var xGtY = x != null && y != null && x > y; var xLtEq0 = x == null || x < 0; - (x = 0) < random() || x > 10; + (x = 0) < rand() || x > 10; return x + 0; } @@ -146,16 +166,17 @@ fun test10(): int { return x + y; } +@method_id(111) fun test11() { - var [x, y] = [getNullableInt(), getNullableInt()]; - if (random()) { return x == null || y == null ? -1 : x + y; } + var [x, y,] = [getNullableInt(), getNullableInt()]; + if (eq(10) < 0) { return x == null || y == null ? -1 : x + y; } if (true && (x == null || y == null) && !!true) { return 0; } return x + y; } fun test12() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random() ? x == null || y == null : x == null || y == null) { return -1; } + if (rand() ? x == null || y == null : x == null || y == null) { return -1; } __expect_type(x, "int"); __expect_type(y, "int"); return x + y; @@ -177,7 +198,7 @@ fun test14() { x = 0; } if (y == null) { - if (random()) { return 0; } + if (rand()) { return 0; } else { y = 0; } } return x + y; @@ -247,45 +268,45 @@ fun test24(x: int?) { fun test25() { var x = (getNullableInt(), getNullableInt(), getNullableInt()); - x.0 = x.2 = random(); + x.0 = x.2 = rand(); return (x.0) + ((x.2)); } fun test26() { var x = [getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()]; - if (~(x.0 = random())) { return; } - if ((x.1 = random()) < (x.2 = random())) { return; } - else if (!(x.2 <=> (x.3 = random()))) { return; } - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - if ((x.7 = random()) as int) { return; } - if (((((x.8 = random()) != null)))) { return; } - if ([x.1, (x.9 = random())!].1) { return; } + if (~(x.0 = rand())) { return; } + if ((x.1 = rand()) < (x.2 = rand())) { return; } + else if (!(x.2 <=> (x.3 = rand()))) { return; } + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + if ((x.7 = rand()) as int) { return; } + if (((((x.8 = rand()) != null)))) { return; } + if ([x.1, (x.9 = rand())!].1) { return; } val result = x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test27() { var (x, _) = ([getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()], []); - +(x.0 = random()); - x.0 += [((x.1 = random()) < (x.2 = random() + x.1)) as int].0; - !(x.2 <=> (x.3 = random() + x.2)); - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - (x.7 = random()) as int; - (((((x.8 = random()) != null)))); - [x.1, (x.9 = random())!].1; + +(x.0 = rand()); + x.0 += [((x.1 = rand()) < (x.2 = rand() + x.1)) as int].0; + !(x.2 <=> (x.3 = rand() + x.2)); + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + (x.7 = rand()) as int; + (((((x.8 = rand()) != null)))); + [x.1, (x.9 = rand())!].1; return x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test28() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type((x.0 = random(), x.0 += (x.1 = random()) as int, !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int), + __expect_type((x.0 = rand(), x.0 += (x.1 = rand()) as int, !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int), "(int, int, bool, int)"); } fun test29() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type([x.0 = random(), ((x.0 += (x.1 = random()) as int)), !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int], + __expect_type([x.0 = rand(), ((x.0 += (x.1 = rand()) as int)), !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int], "[int, int, bool, int]"); } @@ -342,8 +363,8 @@ fun test33(): int { fun test34() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random()) { throw (x = 1, y = 2); } - else { throw (x = 3, y = (1, getNullableInt()!).1); } + if (rand()) { throw (x = 1, y = 2) } + else { throw (x = 3, y = (1, getNullableInt()!).1) } return x + y; } @@ -396,8 +417,8 @@ fun test38() { assignNull2(mutate t.0, mutate t.1); __expect_type(t.0, "int?"); __expect_type(t.1, "slice?"); - t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; - return t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; + t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; + return t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; } @method_id(139) @@ -570,7 +591,7 @@ fun test54() { x1 = getNullableInt(); __expect_type(x1, "int?"); assignToNullableInt(mutate x2, 5); - if (random()) { x3.assignToNullableInt(5); } + if (rand()) { x3.assignToNullableInt(5); } x11 = 10; assignToInt(mutate x12, 5); } while (x1 != null); @@ -579,6 +600,7 @@ fun test54() { __expect_type(x3, "int?"); } +@noinline fun eq55(v: T) { return v; } fun test55() { @@ -591,7 +613,7 @@ fun test55() { // (checked via codegen) eq55(x); __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring - x = random() ? 1 : null; + x = rand() ? 1 : null; } __expect_type(x, "int?"); } @@ -643,6 +665,56 @@ fun test59() { __expect_type(x1, "int"); } +@method_id(160) +fun test60() { + var cc1 = match (val a: int | slice = 0 + 10 + 90) { + int => a + 1, + }; + var cc2 = match (var a = (0 + 10 + 90) as int | slice) { + int => (a = 108) + 1, + slice => 120, + }; + return (cc1, cc2); +} + + +@method_id(161) +fun test61(p: Point) { + var p2: Point? = p; + __expect_type(p2, "Point"); + if (p.x < 10) { + assignNullTo(mutate p2); + } + __expect_type(p2, "Point?"); + return (p2, p2!); +} + +fun test62() { + var p = PointOfNulls { x: 1, y: null, sub: null }; + __expect_type(p.x, "int?"); + __expect_type(p.sub, "Point?"); + if (p.sub == null) { + p.x! > 10 ? p.sub = { x: 1, y : 2 } : Point { x: 3, y: 4 }; + __expect_type(p.sub, "Point?"); + p.x! > 10 ? p.sub = { x: 1, y : 2 } : p.sub = eq({ x: 3, y: 4 }); + } + __expect_type(p.sub, "Point"); + __expect_type(p!!.sub!!, "Point"); + p.sub.x + p.x! + p.y!; + return rand() ? null : p; +} + +fun test63() { + __expect_type(test62, "() -> PointOfNulls?"); + var r = test62(); + __expect_type(r!.sub, "Point?"); + if (r != null) { + r.sub = { x: 1, y: 2 }; + } + __expect_type(r!.sub, "Point?"); + r!.sub = { x: 1, y: 2 }; + __expect_type(r!.sub, "Point"); +} fun main(x: int?): int { @@ -651,6 +723,7 @@ fun main(x: int?): int { /** @testcase | 0 | 1 | 1 +@testcase | 111 | | 10 @testcase | 123 | | 7 @testcase | 124 | 4 | 6 @testcase | 124 | null | 15 @@ -661,18 +734,29 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 -1 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 -1 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 -1 +@testcase | 145 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 typeid-4 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 +@testcase | 160 | | 101 109 +@testcase | 161 | 9 9 | (null) (null) 0 (null) (null) +@testcase | 161 | 19 0 | 19 0 typeid-1 19 0 -@stderr warning: expression of type `int` is always not null, this condition is always true +@stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; -@fif_codegen eq55 PROC:<{ -@fif_codegen eq55 PROC:<{ +@fif_codegen eq55() PROC:<{ +@fif_codegen eq55() PROC:<{ + +@fif_codegen +""" + test60() PROC:<{ + 101 PUSHINT // cc1 + 109 PUSHINT // cc1 cc2 + }> +""" */ diff --git a/tolk-tester/tests/some-tests-1.tolk b/tolk-tester/tests/some-tests-1.tolk new file mode 100644 index 000000000..42bdc773a --- /dev/null +++ b/tolk-tester/tests/some-tests-1.tolk @@ -0,0 +1,127 @@ +fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + var D: int = a * d - b * c; + var Dx: int = e * d - b * f; + var Dy: int = a * f - e * c; + return (Dx / D, Dy / D); +} + +@method_id(101) +fun testDivMod(x: int, y: int) { + return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); +} + +@deprecated +fun twice(f: (int) -> int, x: int) { + return f (f (x)); +} + +fun sqr(x: int) { + return x * x; +} + +@method_id(102) +fun testCallVar(x: int): int { + var f = sqr; + return twice(f, x) * f(x); +} + +@method_id(103) +fun testCallDirect(x: int): int { + return twice(sqr, x) * sqr(x); +} + +@method_id(104) +fun testLoop(x: int): int { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +@method_id(105) +fun test5(id: int): (int, int) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} + +@method_id(106) +fun test6(x: int): int { + var i: int = 0; + do { + i = i + 1; + if (i > 5) { + return 1; + } + var f: bool = (i * i == 64); + } while (!f); + return -1; +} + +@method_id(107) +fun test7(y: int): int { + var x: int = 1; + if (y > 0) { + return 1; + } + return x > 0 ? -1 : 0; +} + +@method_id(108) +fun test8(y: int): int { + if (y > 0) { + return 1; + } + return 2; +} + +@method_id(109) +fun test9(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + + +/** + method_id | in | out +@testcase | 0 | 1 1 1 -1 10 6 | 8 2 +@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +@testcase | 101 | 112 3 | 37 1 1 37 33 6 +@testcase | 102 | 3 | 729 +@testcase | 102 | 10 | 1000000 +@testcase | 103 | 3 | 729 +@testcase | 103 | 10 | 1000000 +@testcase | 104 | 1 | 0 +@testcase | 104 | 2 | 1 +@testcase | 104 | 5 | 5 +@testcase | 104 | 19 | 20 +@testcase | 104 | 27 | 111 +@testcase | 104 | 100 | 25 +@testcase | 105 | 0 | 5 6 +@testcase | 105 | 4 | 5 6 +@testcase | 105 | 11 | 22 33 +@testcase | 106 | 0 | 1 +@testcase | 107 | 10 | 1 +@testcase | 107 | -5 | -1 +@testcase | 108 | 10 | 1 +@testcase | 108 | -5 | 2 +@testcase | 109 | 1 | -2 +@testcase | 109 | 5 | -6 +*/ diff --git a/tolk-tester/tests/a10.tolk b/tolk-tester/tests/some-tests-2.tolk similarity index 66% rename from tolk-tester/tests/a10.tolk rename to tolk-tester/tests/some-tests-2.tolk index 9d24f38da..9a4cb8cd0 100644 --- a/tolk-tester/tests/a10.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-lowlevel" -fun pair_first(p: [X, Y]): X asm "FIRST"; +fun [X, Y].pair_first(self): X asm "FIRST"; fun one(dummy: tuple?) { return 1; @@ -37,14 +37,14 @@ fun test88(x: int) { @method_id(89) fun test89(last: int): (int, int, int, int) { var t: tuple = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush(2); - t.tuplePush(3); - t.tuplePush(last); - return (t.tupleAt(0), t.tupleAt(t.tupleSize() - 1), t.tupleFirst(), t.tupleLast()); + t.push(1); + t.push(2); + t.push(3); + t.push(last); + return (t.get(0), t.get(t.size() - 1), t.first(), t.last()); } -@pure fun get10() { return 10; } +@pure @noinline fun get10() { return 10; } @method_id(91) fun touchCodegen2() { @@ -56,25 +56,25 @@ fun touchCodegen2() { @method_id(92) fun testDumpDontPolluteStack() { var f = get10(); - f.debugPrint(); - debugPrint(10); + debug.print(f); + debug.print(10); var s = "asdf"; - s.debugPrintString(); - debugDumpStack(); - debugPrintString("my"); - return (f, getRemainingBitsCount(s)); + debug.printString(s); + debug.dumpStack(); + debug.printString("my"); + return (f, s.remainingBitsCount()); } @method_id(93) fun testStartBalanceCodegen1() { - var t = getMyOriginalBalanceWithExtraCurrencies(); + var t = contract.getOriginalBalanceWithExtraCurrencies(); var first = t.pair_first(); return first; } @method_id(94) fun testStartBalanceCodegen2() { - var first = getMyOriginalBalance(); + var first = contract.getOriginalBalance(); return first; } @@ -89,6 +89,15 @@ fun test95() { return (cur, next); } +struct Point { + x: int, y: int +} + +@method_id(96) +fun test96(p: Point) { + debug.print(p); // works ok with non-1 stack width, checked via codegen +} + /** method_id | in | out @testcase | 0 | 101 15 | 100 1 @@ -105,15 +114,14 @@ fun test95() { @fif_codegen """ - touchCodegen2 PROC:<{ - // - get10 CALLDICT // f + touchCodegen2() PROC:<{ + get10() CALLDICT // f }> """ @fif_codegen """ - testDumpDontPolluteStack PROC:<{ + testDumpDontPolluteStack() PROC:<{ ... DUMPSTK x{6d79} PUSHSLICE // f s '5 @@ -124,8 +132,7 @@ fun test95() { @fif_codegen """ - testStartBalanceCodegen1 PROC:<{ - // + testStartBalanceCodegen1() PROC:<{ BALANCE // t FIRST // first }> @@ -133,8 +140,7 @@ fun test95() { @fif_codegen """ - testStartBalanceCodegen2 PROC:<{ - // + testStartBalanceCodegen2() PROC:<{ BALANCE FIRST // first }> @@ -142,18 +148,25 @@ fun test95() { @fif_codegen """ - test95 PROC:<{ + test95() PROC:<{ ... - next GETGLOB // g_next + $next GETGLOB // g_next 3 PUSHINT // g_next '14=3 4 PUSHINT // g_next '14=3 '15=4 5 PUSHINT // g_next '14=3 '15=4 '16=5 TRIPLE // '10 '11 SWAP - cur SETGLOB - next SETGLOB - cur GETGLOB // g_cur - next GETGLOB // g_cur g_next + $cur SETGLOB + $next SETGLOB + $cur GETGLOB // g_cur + $next GETGLOB // g_cur g_next + }> +""" + +@fif_codegen +""" + test96() PROC:<{ + s1 DUMP s0 DUMP 2 BLKDROP }> """ */ diff --git a/tolk-tester/tests/w2.tolk b/tolk-tester/tests/some-tests-3.tolk similarity index 91% rename from tolk-tester/tests/w2.tolk rename to tolk-tester/tests/some-tests-3.tolk index 728b18d3f..7208295fe 100644 --- a/tolk-tester/tests/w2.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -15,6 +15,7 @@ fun main(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); } +@noinline fun f(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), @@ -30,5 +31,5 @@ fun f(cs: slice) { @testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 @testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 -@code_hash 58474889199998908444151060994149070836199913191952040273624197630531731101157 +@code_hash 110494322917670257289501183090765859207486616122530510808569169535563021169176 */ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk index 8fae6d5db..11c34f98d 100644 --- a/tolk-tester/tests/special-fun-names.tolk +++ b/tolk-tester/tests/special-fun-names.tolk @@ -4,6 +4,23 @@ fun onRunTickTock() { return -2; } fun onSplitPrepare() { return -3; } fun onSplitInstall() { return -4; } +// 'execute' and many other names are reserved words in Fift, +// but since they are codegenerated like `execute()` (it's a valid Fift name!), no collisions +@noinline +fun execute() { return 123 } +@noinline +fun `PUSHINT`() { return 456 } +@noinline +fun `@havebits`() { return 789 } + +global swap: int; + +@method_id(101) +fun test101() { + swap = 1; + return execute() + `PUSHINT`() + `@havebits`() + swap; +} + /** @experimental_options remove-unused-functions @@ -12,13 +29,18 @@ fun onSplitInstall() { return -4; } @testcase | -2 | | -2 @testcase | -3 | | -3 @testcase | -4 | | -4 +@testcase | 101 | | 1369 @fif_codegen """ - 0 DECLMETHOD onInternalMessage - -1 DECLMETHOD onExternalMessage - -2 DECLMETHOD onRunTickTock - -3 DECLMETHOD onSplitPrepare - -4 DECLMETHOD onSplitInstall + 0 DECLMETHOD onInternalMessage() + -1 DECLMETHOD onExternalMessage() + -2 DECLMETHOD onRunTickTock() + -3 DECLMETHOD onSplitPrepare() + -4 DECLMETHOD onSplitInstall() """ + +@fif_codegen DECLPROC execute() +@fif_codegen execute() CALLDICT +@fif_codegen DECLGLOBVAR $swap */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk new file mode 100644 index 000000000..99a422161 --- /dev/null +++ b/tolk-tester/tests/strings-tests.tolk @@ -0,0 +1,82 @@ +get fun ascii_slice(): slice { + return"string"; +} + +get fun raw_slice(): slice { + return stringHexToSlice("abcdef"); +} + +get fun addr_slice(): address { + return address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); +} + +get fun string_hex(): int { + return stringToBase256("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"); +} + +get fun string_minihash(): int { + return stringSha256_32("transfer(slice, int)"); +} + +get fun string_maxihash(): int { + return stringSha256("transfer(slice, int)"); +} + +get fun string_crc32(): int { + return stringCrc32("transfer(slice, int)"); +} + +get fun string_crc16(): int { + return stringCrc16("transfer(slice, int)"); +} + +@pure +fun newc(): builder +asm "NEWC"; +fun builder.endcs(self): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): int +asm "SDEQ"; + +fun calcSliceBytes(slice: slice): tuple { + var t = createEmptyTuple(); + var n = 0; + while (n = slice.remainingBitsCount()) { + t.push(slice.loadUint(min(8, n))); + } + return t; +} + +fun main() { + var s_ascii: slice = ascii_slice(); + var s_raw: slice = raw_slice(); + var s_addr: address = addr_slice(); + var i_hex: int = string_hex(); + var i_mini: int = string_minihash(); + var i_maxi: int = string_maxihash(); + var i_crc32: int = string_crc32(); + var i_crc16: int = string_crc16(); + assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; + assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; + assert(sdeq(s_addr as slice, newc().storeUint(4, 3).storeInt(-1, 8) + .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); + assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; + assert(i_mini == 0x7a62e8a8) throw 105; + assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; + assert(i_crc32 == 2235694568 && i_crc32 == 0x8541fde8) throw 107; + assert(i_crc16 == 52550 && i_crc16 == 0xCD46) throw 108; + return 0; +} + +@method_id(101) +fun test1() { + return calcSliceBytes("ABCD"); +} + +/** +@testcase | 0 | | 0 +@testcase | 101 | | [ 65 66 67 68 ] + +@code_hash 52991558142486159683575375539594116045551936710713942466413832119934760926753 +*/ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk new file mode 100644 index 000000000..8eba3b427 --- /dev/null +++ b/tolk-tester/tests/struct-tests.tolk @@ -0,0 +1,684 @@ +struct JustInt { + value: int; +} + +struct JustIntWrapper { + int: JustInt; +} + +type MInt = int +type JustIntAlias = JustInt +type JustIntAlias2 = JustIntAlias; + +struct Storage { + owner: User, + lastPoint: Point, +} + +struct Point { + x: int + y: MInt +} + +struct User { + id: int + name: slice +} + +struct OuterStorage { + st: Storage, + stMaybe: Storage? +} + +struct WithTensorInside { + coords: (int, int) + tup: [int, int] + otherCoords: (int, int)? + otherTup: tuple +} + +fun sumCoords(p: Point) { + return p.x + p.y; +} + +fun Point.sumCoords(self) { + return self.x + self.y; +} + +fun getStorage1(x: int): Storage { + val owner = User { id: 1, name: "u" }; + return { + owner, + lastPoint: { x, y: 10, }, + }; +} + +fun generatePoint(x: int, y: int): Point { + return { x, y }; +} + +fun assignCoords(mutate p: Point) { + p.x = 10; + p.y = 20; +} + +fun Point.assignCoords(mutate self) { + self.x = 10; + self.y = 20; +} + +@method_id(101) +fun test1() { + var i1 = JustInt { value: 1 }; + var i2: JustInt = JustInt { value: 2, }; + var i3: JustInt = { `value`: 3 }; + __expect_type(i3, "JustInt"); + __expect_type(i3!, "JustInt"); + return (i1, i2, i3, [i1, JustInt{value:5}]); +} + +@method_id(102) +fun test2() { + var i1 = JustIntAlias2 { value: 1 }; + var i2: JustInt = JustIntAlias2 { value: 2, }; + var i3: JustIntAlias2 = { `value`: 3 }; + __expect_type(i1.value, "int"); + __expect_type(i1.value + i2.value + i3.value, "int"); + __expect_type(i3, "JustIntAlias2"); + __expect_type([i2, i3], "[JustInt, JustIntAlias2]"); + __expect_type([i2] as [JustIntAlias], "[JustIntAlias]"); + return (i1, i2, i3, [i1, JustIntAlias{value:5}]); +} + +@method_id(103) +fun test3() { + return generatePoint(5, 6); +} + +@method_id(104) +fun test4() { + var p: Point = { x: 10, y: 20 }; + assert(sizeof(p) == 2, 100); + return (p == null, p, p = {x:30,y:40}, p != null); +} + +@method_id(105) +fun test5() { + var b = PointAlias { y: 20, x: 10 }; + var p: PointAlias = Point { x: b.x, y: b.y }; + p.x += 5; + return (sumCoords(p), p.y += 10, p.sumCoords()); +} + +struct BacktickNames { + `my()id`: MInt, + `100500`: int; +} + +@method_id(106) +fun test6() { + val b: BacktickNames = { `100500`: 20, `my()id`: 10 }; + var p: Point = { x: 0, y: 0 }; + p.x += b.`my()id`; + p.y += b.100500; + return p; +} + +@method_id(107) +fun test7() { + var s = getStorage1(5); + s.owner.name = beginCell().storeInt(s.lastPoint.x, 32).endCell().beginParse(); + var s2 = s; + assignCoords(mutate s.lastPoint); + return (s.owner.name.loadInt(32), s2.owner.name.loadInt(32), s.lastPoint, s2.lastPoint.sumCoords()); +} + +struct Empty +struct OuterEmpty { nothing: EmptyAlias } +type EmptyAlias = Empty + +struct EmptyGeneric1; +struct EmptyGeneric2 + +@method_id(108) +fun test8() { + var e1: Empty = {}; + var o1: OuterEmpty = { nothing: e1 }; + var o2 = OuterEmpty { nothing: {} }; + __expect_type(o2.nothing, "EmptyAlias"); + var o3 = Empty{} as Empty?; + var o4 = null as Empty?; + var o5: EmptyAlias? = o3!; + var o6: OuterEmpty? = { nothing: {} }; + var o7 = o3! as EmptyAlias?; + var o8 = o6 as OuterEmpty?; + __expect_type(o3, "Empty?"); + __expect_type(o6, "OuterEmpty"); + __expect_type(o8, "OuterEmpty?"); + __expect_type(EmptyGeneric2{}, "EmptyGeneric2"); + return (e1, Empty{}, EmptyAlias{}, o2, o1, 777, o3, 777, o4, 777, o5, 777, o6, 777, o7, 777, o8, 777, o3!, 777); +} + +fun maxCoord(p: Point): int8 { + return p.x > p.y ? p.x : p.y; +} + +fun Point.maxCoord(self): int8 { + return self.x > self.y ? self.x : self.y; +} + +@method_id(109) +fun test9() { + var p = Point { x: 80, y: 100 }; + p.assignCoords(); + return (maxCoord({y: 70, x: 30}), maxCoord(generatePoint(30, 20)), maxCoord(p), p.maxCoord(), Point{x:-80,y:-80}.maxCoord()); +} + +@method_id(110) +fun test10(notNull: bool) { + var p: Point? = null; + __expect_type(p, "null"); + if (notNull) { + p = { x: 1, y: 2 }; + __expect_type(p, "Point"); + __expect_type(p.x, "int"); + __expect_type(p.y, "MInt"); + __expect_type(p.x + p.y, "int"); + assignCoords(mutate p); + } + __expect_type(p, "Point?"); + return p; +} + +@method_id(111) +fun test11(notNull: bool) { + var os: OuterStorage = { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: notNull ? { owner: { id: 1, name: "" }, lastPoint: { x: 1, y: 1 } } : null, + }; + if (os.stMaybe == null) { + os!.st!.lastPoint!.y = 2; + os.stMaybe = { owner: { id: 2, name: "" }, lastPoint: { x: 2, y: 3 } }; + } else { + os.stMaybe.owner.id += 1; + os.stMaybe.lastPoint = { x: 3, y: 4 }; + } + if (!notNull) { + assignCoords(mutate os.stMaybe.lastPoint); + os.stMaybe.lastPoint.assignCoords(); + } + return (os.st.lastPoint, os.stMaybe.lastPoint, os.stMaybe.lastPoint.sumCoords()); +} + +@method_id(112) +fun test12(notNull: bool): Point? { + return notNull ? { x: 1, y : 2 } : null; +} + +@method_id(113) +fun test13() { + return null as User?; +} + +@method_id(114) +fun test14() { + var p: Point? = { y: 2, x: 1 }; + __expect_type(p, "Point"); + __expect_type(p as Point, "Point"); + __expect_type(p as Point?, "Point?"); + return p; +} + +@method_id(115) +fun test15() { + var os = OuterStorage { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: null, + }; + __expect_type(os.stMaybe, "Storage?"); + return os.stMaybe; +} + +fun pushZero(mutate t: tuple) { + t.push(0); +} + +@method_id(116) +fun test16(): WithTensorInside? { + var wt: WithTensorInside = { + coords: (1, 2), + tup: [3, 4], + otherCoords: null, + otherTup: createEmptyTuple(), + }; + wt.coords = (-1, -2); + wt.coords.1 -= 2; + wt.tup.0 += 5; + wt.otherCoords!.0 = 7; + wt.otherTup.push(0); + pushZero(mutate wt.otherTup); + (wt.otherTup.1, wt.otherTup.0) = (11, 10); + return wt; +} + +@method_id(117) +fun test17(x: JustInt?) { + var w1: JustIntWrapper = { int: { value: x == null ? -1 : x.value } }; + var w2: JustIntWrapper? = x == null ? x : { int: { value: x.value } }; + return (x, x!, x!.value, w1, w2); +} + +@noinline +fun sumXY

(point: P) { return point.x + point.y; } + +@method_id(118) +fun test18() { + var p = `Point` { x: 8, `y`: 9 } as `PointAlias`?; + if (p != null) { + return sumXY(p); + } + return sumXY(p = getStorage1(80).lastPoint); +} + +global gPoint: Point; +global gPointN: Point?; + +@method_id(119) +fun test19() { + gPoint = { x: 1, y: 2 }; + assignCoords(mutate gPoint); + gPointN = getStorage1(9).lastPoint; + return (gPoint, gPointN, gPointN!.x); +} + +@method_id(120) +fun test20(setNull: bool) { + gPoint = generatePoint(8, 9); + gPointN = setNull ? null : gPoint; + if (!setNull) { + gPoint.x = gPointN!.y = 80; + } else { + (gPoint as Point?)!.y += 5; + } + return (gPoint, gPointN); +} + +global gJustInt: JustInt; +global gJustIntN: JustInt?; + +@method_id(121) +fun test21(setNull: bool) { + var value = 8; + gJustInt = { value }; + gJustIntN = setNull ? null : { value: value += 9 }; + (gJustInt.value, value) = (value, gJustInt.value); + return (gJustInt, gJustIntN, value, value * (gJustIntN == null ? 1 : gJustIntN!.value)); +} + +fun asm_func(x1: JustInt, x2: int, x3: JustIntAlias, x4: JustIntAlias, x5: int, x6: JustInt, x7: int): + (int, JustInt, int, JustInt, int, int, JustIntAlias2) + asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; + +fun JustInt.someOp(mutate self, y: int): int { + val (newX, newY) = (self.value + y, y * 10); + self = { value: newX }; + return newY; +} + +@method_id(122) +fun test22(x: JustInt) { + return asm_func(x, x.value += 1, x, x, x.someOp(x.value / 20), x, x.value = x.value * 2); +} + +@method_id(123) +fun test23(p: Point) { + var complex = p as Point | (JustInt | Storage); + return match (complex) { + JustInt => 10 * complex.value, + PointAlias => 20, + Storage => 30 + complex.lastPoint.x, + }; +} + +@method_id(124) +fun test24(x: JustInt | JustIntAlias2 | JustIntAlias): JustInt { + return match (x) { + JustIntAlias => x + }; +} + +fun getPointOrStorage(getPoint: bool): Point | Storage { + return getPoint + ? Point { x: 10, y: 20 } + : Storage { owner: { id: 10, name: "" }, lastPoint: { x: 30, y: 40 } }; +} + +@method_id(125) +fun test25() { + var r1 = getPointOrStorage(true); + if (r1 !is Storage && r1 !is builder && r1 != null) { + match (val d = getPointOrStorage(false)) { + Point => { + r1.x += d.x; + r1.y += d.y; + } + Storage => { + r1.x += d.lastPoint.x; + r1.y += d.lastPoint.y; + } + } + } + return r1; +} + +fun acceptSomeUnion(r: Point | int | builder) { + return (r is Point) ? r.x + r.y : (r is int) ? r : null; +} + +@method_id(126) +fun test26() { + __expect_type(acceptSomeUnion, "(Point | int | builder) -> int?"); + return acceptSomeUnion({ x: 8, y: 10 })! + acceptSomeUnion(100)!; +} + +struct Has2EmptyNullable { + f1: Empty?; + f2: Empty? +} + +fun get2EmptyNullable(null1: bool, null2: bool): Has2EmptyNullable { + return { f1: null1 ? null : {}, f2: null2 ? null : {} }; +} + +@method_id(127) +fun test27() { + var (t1, t2) = (get2EmptyNullable(true, true), get2EmptyNullable(false, false)); + var (t3, t4) = (t1 as Has2EmptyNullable?, null as Has2EmptyNullable?); + return (t1, t2, 777, t1.f1 == null, t1.f2 == null, t2.f1 == null, t2.f2 == null, 777, t3, 777, t3!, 777, t4); +} + +fun getEmptyOrIntInt(getEmpty: bool): Empty | (int, int) { + return getEmpty ? {} : (1, 2); +} + +@method_id(128) +fun test28(getEmpty: bool) { + var t1: Empty | (int, int) = getEmptyOrIntInt(getEmpty); + var t2 = (getEmpty ? Empty{} : Has2EmptyNullable{ f1: null, f2: {} }) as EmptyAlias | Has2EmptyNullable | null; + return (t1, t1 is Empty, t1 is (int, int), t1 is builder, 777, t2, t2 is Empty, t2 is Has2EmptyNullable, t2 == null); +} + +struct Empty1; +struct Empty2; + +@method_id(129) +fun test29() { + var c = (null as Empty1 | Empty2?); + if (c == null) { + return c; + } + return null; +} + +@method_id(130) +fun test30(is1: bool) { + var e: Empty1 | Empty2 = is1 ? Empty1{} : Empty2 {}; + var n: Empty1 | int | Empty2 = 4; + var c: Empty1 | Empty2? = null; + if (is1) { + n = e; + } else { + c = e; + } + __expect_type(test29(), "null"); + return (e, 777, n, 777, c, 777, (test29() as Empty1? | Empty2), (test29() as Empty1 | Empty2 | null)!); +} + +struct IntOrNull { + e: Empty; + x: int?; +} + +@method_id(131) +fun test31() { + var s1 = { x: 5, e: {} } as IntOrNull?; + var s2 = { e: {}, x: null } as IntOrNull?; + var s3 = null as IntOrNull?; + return (s1, s2, s3, 777, s1 == null, s2 == null, s3 == null, 777, s1!.x == null, s2!.x == null); +} + +struct PointDef0 { + x: int = 0, + y: int = 0, +} + +@method_id(132) +fun test32() { + var p1: PointDef0 = { x: 10, y: 20 }; + var p2: PointDef0 = { x: 10 }; + var p3: PointDef0 = { y: 20 }; + var p4: PointDef0 = { }; + return (p1, p2, p3, p4, PointDef0{}); +} + +struct WithDefaults { + f1: (bool, int) = (true, 0), + f2: int, + f3: slice? = stringHexToSlice("010203"), + f4: PointDef0? = null, + f5: int32 | int64 = 0 as int32, +} + +@method_id(133) +fun test33(): WithDefaults { + var w1: WithDefaults = { f2: 0 }; + assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.remainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; + var w2: WithDefaults? = { f1: (false, 55), f2: 10, f5: 8 as int64 }; + assert(w2.f1.0 != true && w2.f3!.remainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; + var w3: (int, WithDefaults) = (0, { f2: 7, f4: {y: 20} }); + assert(w3.1.f4 != null && w3.1.f4.x == 0 && w3.1.f4.y == 20) throw 100; + return { f2: 5, f3: null }; +} + +struct WithNever { + f1: int; + f2: never; + f3: int; +} + +@method_id(134) +fun test34() { + var o1: WithNever = { f1: 10, f3: 20 }; // f2 is `never`, it can be omitted + __expect_type(o1.f2, "never"); + return o1; +} + +@pure @noinline fun getXPure() { return 1; } +@pure @noinline fun getYPure() { return 2; } +global t_impure: tuple; +@noinline fun getXImpure() { t_impure.push(1); return 1; } +@noinline fun getYImpure() { t_impure.push(2); return 2; } + +@method_id(135) +fun test35() { + var p: Point = { + y: getYPure(), + x: getXPure(), + }; + return p; +} + +@method_id(136) +fun test36() { + t_impure = createEmptyTuple(); + var p: Point = { + y: getYImpure(), + x: getXImpure(), + }; + return (p, t_impure); +} + +@method_id(137) +fun test37() { + var num = 0; + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +@method_id(138) +fun test38(num: int) { + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +struct TwoPoints { + p1: Point; + p2: Point; +} + +@method_id(139) +fun test39(): (TwoPoints, int) { + var cs = stringHexToSlice("0102030405"); + return ({ + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p1: { x: cs.loadUint(8), y: cs.loadUint(8) }, + }, cs.remainingBitsCount()); +} + +@method_id(140) +fun test40() { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p1: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + } +} + +@method_id(141) +fun test41(rev: bool) { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p2: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + p1: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + } +} + +fun main(x: int8, y: MInt) { + __expect_type(PointAlias{x,y}, "Point"); + __expect_type(Point{x,y} as Point, "Point"); + __expect_type(test3(), "Point"); + __expect_type(maxCoord, "(Point) -> int8"); +} + +type PointAlias = Point; + +/** +@testcase | 101 | | 1 2 3 [ 1 5 ] +@testcase | 102 | | 1 2 3 [ 1 5 ] +@testcase | 103 | | 5 6 +@testcase | 104 | | 0 10 20 30 40 -1 +@testcase | 105 | | 35 30 45 +@testcase | 106 | | 10 20 +@testcase | 107 | | 5 5 10 20 15 +@testcase | 108 | | 777 typeid-3 777 0 777 777 777 typeid-3 777 typeid-4 777 777 +@testcase | 109 | | 70 30 20 20 -80 +@testcase | 110 | 0 | (null) (null) 0 +@testcase | 110 | -1 | 10 20 typeid-5 +@testcase | 111 | 0 | 0 2 10 20 30 +@testcase | 111 | -1 | 0 0 3 4 7 +@testcase | 112 | 0 | (null) (null) 0 +@testcase | 112 | -1 | 1 2 typeid-5 +@testcase | 113 | | (null) (null) 0 +@testcase | 114 | | 1 2 +@testcase | 115 | | (null) (null) (null) (null) 0 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-7 +@testcase | 117 | 5 | 5 5 5 5 5 +@testcase | 117 | null | (null) (null) (null) -1 (null) +@testcase | 118 | | 17 +@testcase | 119 | | 10 20 9 10 typeid-5 9 +@testcase | 120 | 0 | 80 9 8 80 typeid-5 +@testcase | 120 | -1 | 8 14 (null) (null) 0 +@testcase | 121 | 0 | 17 17 8 136 +@testcase | 121 | -1 | 8 (null) 8 8 +@testcase | 122 | 100 | 101 50 106 212 100 101 101 +@testcase | 124 | 66 | 66 +@testcase | 125 | | (null) (null) 40 60 typeid-5 +@testcase | 126 | | 118 +@testcase | 127 | | 0 0 typeid-3 typeid-3 777 -1 -1 0 0 777 0 0 typeid-10 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) typeid-3 -1 0 0 777 (null) (null) typeid-3 -1 0 0 +@testcase | 128 | 0 | 1 2 typeid-2 0 -1 0 777 0 typeid-3 typeid-10 0 -1 0 +@testcase | 129 | | (null) +@testcase | 130 | -1 | typeid-12 777 (null) typeid-12 777 0 777 0 0 +@testcase | 130 | 0 | typeid-11 777 4 1 777 typeid-11 777 0 0 +@testcase | 131 | | 5 typeid-13 (null) typeid-13 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 +@testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 +@testcase | 134 | | 10 20 +@testcase | 135 | | 1 2 +@testcase | 136 | | 1 2 [ 2 1 ] +@testcase | 137 | | 48 5 50 +@testcase | 138 | 3 | 78 8 80 +@testcase | 139 | | 3 4 2 1 8 +@testcase | 140 | | 2 1 4 3 +@testcase | 141 | -1 | 4 3 2 1 +@testcase | 141 | 0 | 3 4 1 2 + +@fif_codegen +""" + test6() PROC:<{ + 10 PUSHINT // p.x + 20 PUSHINT // p.x p.y + }> +""" + +@fif_codegen +""" + sumXY() PROC:<{ // point.x point.y + ADD // '2 + }> +""" +@fif_codegen_avoid sumXY + +@fif_codegen +""" + test23() PROC:<{ // p.x p.y + 2DROP + 20 PUSHINT // '10=20 + }> +""" + +@fif_codegen +""" + test29() PROC:<{ + PUSHNULL // '5 + }> +""" + +@fif_codegen +""" + test35() PROC:<{ // + getXPure() CALLDICT // '2 + getYPure() CALLDICT // p.x p.y + }> +""" + +@fif_codegen +""" + test36() PROC:<{ // + NIL // '0 + $t_impure SETGLOB // + getYImpure() CALLDICT // '4 + getXImpure() CALLDICT // p.y p.x + $t_impure GETGLOB // p.y p.x g_t_impure + s1 s2 XCHG // p.x p.y g_t_impure + }> +""" + + + */ diff --git a/tolk-tester/tests/ternary-tests.tolk b/tolk-tester/tests/ternary-tests.tolk new file mode 100644 index 000000000..a1e88a8e8 --- /dev/null +++ b/tolk-tester/tests/ternary-tests.tolk @@ -0,0 +1,367 @@ +const TWO = 2; +global gTup: tuple; + +fun get1() { gTup.push(1); return 1; } +fun get2() { gTup.push(2); return 2; } + +struct Point { + x: int; + y: int; +} + +@noinline +fun Point.create0NoInline(): Point { + return { x: 0, y: 0 } +} + +fun Point.create0(): Point { + return { x: 0, y: 0 } +} + +@method_id(101) +fun test101(x: int) { + return (x ? 2 : 1, x == 5 ? TWO : 1); +} + +@method_id(102) +fun test102(x: int?) { + return x == null ? (0!) : ((x)); +} + +@method_id(103) +fun test103(x: bool) { + if (x ? -1 : +1) { + return x ? -2 : +2; + } + throw 123; +} + +@method_id(104) +fun test104(x: (int, int)) { + return x.0 ? true : null; +} + +@method_id(105) +fun test105(getX: bool) { + var p: Point = { x: 10, y: 20 }; + return getX ? p.x : p!.y; +} + +@noinline +fun helper106(glob: bool, t: tuple) { + return glob ? gTup : t; +} + +@method_id(106) +fun test106() { + gTup = createEmptyTuple(); + gTup.push(1); + return helper106(true, createEmptyTuple()); +} + +@noinline +fun helper107(t: (int, Point)) { + return t.0 ? t.0 : t.1.y; +} + +@method_id(107) +fun test107() { + return (helper107((1, {x: 10, y:20})), helper107((0, {x: 10, y: 20}))); +} + +@method_id(108) +fun test108(calcMin: bool) { + var cb = calcMin ? min : max; + return cb(10, 20); +} + +@method_id(109) +fun test109(bigCost: bool) { + return bigCost ? ton("0.05") : (ton("0.001")); +} + +@method_id(110) +fun test110(k: int) { + val c: [int, int, int] = [k, 6, 7]; + return c.0 > 0 ? c.1 : c.2; +} + +@method_id(111) +fun test111(g: bool) { + return g ? (1, 2) : (3, 4); +} + + +@method_id(200) +fun test200(x: int) { + return 10 > 3 ? 200 : 100; +} + +@method_id(201) +fun test201(x: int) { + return (x is slice) ? 200 : x; +} + +@method_id(202) +fun test202(x: int) { + x > 10 ? x : 0; + x < 10 ? x! : -10; + return x; +} + +@method_id(203) +fun test203() { + var xx = false; + if (!xx) { + return xx ? 100 : 200; + } + return 300; +} + + +@method_id(220) +fun test220(p: Point?, getNull: bool) { + if (p == null) { + return getNull ? p : 20; + } + return p.y; +} + +@method_id(221) +fun test221(c: (int, (int,int)?)) { + if (c.1 != null) { + return c.0 > 5 ? c.1.0 : c.1.1; + } + return c.0 > 5 ? 0 : c.1; +} + + +@method_id(300) +fun test300(x: int): tuple | int { + // no condsel, because more that 1 slot + return x > 0 ? x : gTup; +} + +@method_id(301) +fun test301(first: bool) { + var opt1 = (10, 20); + var opt2 = (30, 40); + // no condsel, because not 1 slot arguments + return first ? opt1 : opt2; +} + +@method_id(302) +fun test302(x: int?) { + // no condsel, because not trivial (unary operator) + return x == null ? 0 : -x; +} + +@method_id(303) +fun test303(first: bool) { + gTup = createEmptyTuple(); + // no condsel, because not trivial + return (first ? get1() : get2(), gTup); +} + +@method_id(304) +fun test304(y: bool) { + return y ? Point.create0NoInline().y : 5; +} + +@method_id(305) +fun test305(y: bool) { + return y ? Point.create0().y : 5; +} + + +fun main() {} + +/** +@testcase | 101 | 0 | 1 1 +@testcase | 101 | 5 | 2 2 +@testcase | 101 | 9 | 2 1 +@testcase | 102 | 8 | 8 +@testcase | 102 | null | 0 +@testcase | 103 | -1 | -2 +@testcase | 104 | 1 2 | -1 +@testcase | 104 | 0 2 | (null) +@testcase | 105 | -1 | 10 +@testcase | 106 | | [ 1 ] +@testcase | 107 | | 1 20 +@testcase | 108 | 0 | 20 +@testcase | 109 | -1 | 50000000 +@testcase | 110 | 5 | 6 +@testcase | 110 | -5 | 7 +@testcase | 111 | -1 | 1 2 +@testcase | 111 | 0 | 3 4 + +@testcase | 200 | 20 | 200 +@testcase | 201 | 5 | 5 +@testcase | 202 | 5 | 5 +@testcase | 203 | | 200 + +@testcase | 220 | null null 0 -1 | (null) +@testcase | 220 | null null 0 0 | 20 +@testcase | 220 | 9 8 123 5 | 8 +@testcase | 221 | 10 null null 0 | 0 +@testcase | 221 | 3 null null 0 | (null) +@testcase | 221 | 10 3 4 100 | 3 +@testcase | 221 | 3 3 4 100 | 4 + +@testcase | 300 | 5 | 5 1 +@testcase | 301 | -1 | 10 20 +@testcase | 302 | null | 0 +@testcase | 303 | -1 | 1 [ 1 ] +@testcase | 304 | -1 | 0 +@testcase | 304 | 0 | 5 +@testcase | 305 | 0 | 5 +@testcase | 305 | 0 | 5 + +@fif_codegen +""" + test101() PROC:<{ + DUP + 2 PUSHINT + 1 PUSHINT + CONDSEL + SWAP + 5 EQINT + 2 PUSHINT + 1 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test102() PROC:<{ // x + DUP // x x + ISNULL // x '1 + 0 PUSHINT + ROT // '1 '3=0 x + CONDSEL // '2 + }> +""" + +@fif_codegen +""" + test103() PROC:<{ + -2 PUSHINT + 2 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test105() PROC:<{ // getX + 10 PUSHINT // getX '3=10 + 20 PUSHINT // getX p.x=10 p.y=20 + CONDSEL // '5 + }> +""" + +@fif_codegen +""" + helper106() PROC:<{ + $gTup GETGLOB + SWAP + CONDSEL + }> +""" + +@fif_codegen +""" + helper107() PROC:<{ + NIP + s1 s(-1) PUXC + CONDSEL + }> +""" + +@fif_codegen +""" + test108() PROC:<{ + CONT:<{ + MIN + }> + CONT:<{ + MAX + }> + CONDSEL + 10 PUSHINT + 20 PUSHINT + ROT + 2 1 CALLXARGS + }> +""" + +@fif_codegen +""" + test109() PROC:<{ + 50000000 PUSHINT + 1000000 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test200() PROC:<{ + DROP + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test201() PROC:<{ + // '2 + }> +""" + +@fif_codegen +""" + test202() PROC:<{ + }> +""" + +@fif_codegen +""" + test203() PROC:<{ + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test220() PROC:<{ + s3 POP + 0 EQINT + IFJMP:<{ + 20 PUSHINT + CONDSEL + }> + NIP + }> +""" + +@fif_codegen +""" + test304() PROC:<{ + IF:<{ + Point.create0NoInline() CALLDICT +""" + +@fif_codegen +""" + test305() PROC:<{ + IF:<{ + 0 PUSHINT + }>ELSE<{ + 5 PUSHINT + }> + }> +""" + +*/ diff --git a/tolk-tester/tests/test-math.tolk b/tolk-tester/tests/test-math.tolk index 6b147c776..55c518800 100644 --- a/tolk-tester/tests/test-math.tolk +++ b/tolk-tester/tests/test-math.tolk @@ -921,7 +921,7 @@ fun nrand_f252(): int { var (x, s, t, A, B, r0) = (nan(), 29483 << 236, -3167 << 239, 12845, 16693, 9043); // 4/sqrt(e*Pi) = 1.369 loop iterations on average do { - var (u, v) = (random() / 16 + 1, mulDivRound(random() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) + var (u, v) = (random.uint256() / 16 + 1, mulDivRound(random.uint256() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) var va: int = abs(v); var (u1, v1) = (u - s, va - t); // (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 // Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 @@ -952,7 +952,7 @@ fun nrand_f252(): int { fun nrand_fast_f252(): int { var t: int = -3 << 253; // -6. as fixed252 repeat (12) { - t += random() / 16; // add together 12 uniformly random numbers + t += random.uint256() / 16; // add together 12 uniformly random numbers } return t; } @@ -961,7 +961,7 @@ fun nrand_fast_f252(): int { /// fixed248 random(); @inline fun fixed248_random(): int { - return random() >> 8; + return random.uint256() >> 8; } /// random number with standard normal distribution @@ -979,7 +979,7 @@ fun fixed248_nrand_fast(): int { } @pure -fun tset(mutate self: tuple, idx: int, value: X): void +fun tuple.tset(mutate self, idx: int, value: X): void asm(self value idx) "SETINDEXVAR"; // computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method @@ -1014,12 +1014,12 @@ fun asin_slow_f255(x: int): int { fun test_nrand(n: int): tuple { var t: tuple = createEmptyTuple(); repeat (255) { - t.tuplePush(0); + t.push(0); } repeat (n) { var x: int = fixed248_nrand(); var bucket: int = (abs(x) >> 243); // 255 buckets starting from x=0, each 1/32 wide - var at_bucket: int = t.tupleAt(bucket); + var at_bucket: int = t.get(bucket); t.tset(bucket, at_bucket + 1); } return t; diff --git a/tolk-tester/tests/try-func.tolk b/tolk-tester/tests/try-catch-tests.tolk similarity index 88% rename from tolk-tester/tests/try-func.tolk rename to tolk-tester/tests/try-catch-tests.tolk index 4ac86d966..136c606d2 100644 --- a/tolk-tester/tests/try-func.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -164,6 +164,7 @@ fun test109(): (int, int) { return (g_reg, l_reg); } +@noinline fun alwaysThrow123(): never { throw 123; } @@ -173,6 +174,7 @@ fun alwaysThrowX(x: int): never { else { throw (x, null); } } +@noinline fun anotherNever(throw123: bool): never { if (throw123) { alwaysThrow123(); } alwaysThrowX(456); @@ -224,9 +226,13 @@ fun test111(b: bool) { } } -fun mySetCode(newCode: slice): void +fun mySetCodeAsm(newCode: slice): void asm "SETCODE"; +fun mySetCode(newCode: slice) { + mySetCodeAsm(newCode); +} + fun testCodegen3(numberId: int, paramVal: cell) { if (numberId == -1000) { var cs = paramVal.beginParse(); @@ -236,6 +242,27 @@ fun testCodegen3(numberId: int, paramVal: cell) { paramVal.beginParse(); } +@method_id(112) +fun testCatchUsesVars(x1: int) { + var x2 = x1 * 2; + try { alwaysThrow123(); } + catch (excno) { + var result = x1 + x2 + excno; + return result; + } + return -1; +} + +@method_id(113) +fun testBigExcno() { + try { + throw 2048; + return 10; + } catch (excno) { + return excno; + } +} + fun main() { } @@ -264,13 +291,14 @@ fun main() { @testcase | 110 | 0 | 5 @testcase | 111 | -1 | 123 @testcase | 111 | 0 | 456 +@testcase | 112 | 5 | 138 +@testcase | 113 | | 2048 -@code_hash 57361460846265694653029920796509802052573595128418810728101968091567195330515 +@code_hash 3924051084946061509165180039638830343498397714643311802900659572522552334228 @fif_codegen """ - testCodegen1 PROC:<{ - // x + testCodegen1() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -288,18 +316,17 @@ fun main() { @fif_codegen """ - testCodegen2 PROC:<{ - // x + testCodegen2() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x DROP // - alwaysThrow123 CALLDICT + alwaysThrow123() CALLDICT }> // x 10 LESSINT // '5 IFJMP:<{ // FALSE // '6 - anotherNever CALLDICT + anotherNever() CALLDICT }> // 0 PUSHINT // '8=0 }> @@ -307,8 +334,7 @@ fun main() { @fif_codegen """ - testCodegen3 PROC:<{ - // numberId paramVal + testCodegen3() PROC:<{ // numberId paramVal SWAP -1000 PUSHINT // paramVal numberId '2=-1000 EQUAL // paramVal '3 @@ -320,4 +346,12 @@ fun main() { DROP // }> """ + +@fif_codegen +""" +<{ + 11 PUSHPOW2 + THROWANY +}>CONT +""" */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk new file mode 100644 index 000000000..3bc0e5114 --- /dev/null +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -0,0 +1,224 @@ +import "@stdlib/tvm-dicts.tolk" + +type MIntN = MInt? +type MInt = int +type MInt_v2 = int +type MVoid = void +type Pair2_v1 = | (int, int) +type Pair2_v2 = (MInt, MInt) +type MBool = | bool +type Tuple2Int = [int, int] + +struct Point { x: int; y: int } +type PointAlias = Point; +type PointAlias2 = PointAlias; + +fun rand(): uint256 + asm "RANDU256"; + +fun test1(x: MInt): MVoid { + var y = x; + var z: MInt = 2; + __expect_type(x, "MInt"); + __expect_type(y, "MInt"); + __expect_type(z, "MInt"); + + __expect_type(x + y, "int"); + __expect_type((x + y) as MInt, "MInt"); + __expect_type(x!, "MInt"); + __expect_type(~x, "int"); + __expect_type(x as int, "int"); + __expect_type(x as int8, "int8"); + __expect_type(rand() ? x : y, "MInt"); + __expect_type((x, 1, y), "(MInt, int, MInt)"); + + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + + __expect_type(!x, "bool"); + + __expect_type(x as int?, "int?"); + __expect_type(x as MInt?, "MInt?"); + __expect_type(x as MIntN, "MIntN"); + + __expect_type(PointAlias{x:0,y:0}, "Point"); + __expect_type(PointAlias2{x:0,y:0}, "Point"); + __expect_type(Point{x:0,y:0} as PointAlias, "PointAlias"); + + if (x) { return; } +} + +fun test2() { + __expect_type(test1, "(MInt) -> void"); + __expect_type(test1(1), "void"); +} + +fun test3(x: MIntN, y: MInt?) { + __expect_type(x, "MIntN"); + __expect_type(x!, "MInt"); + __expect_type(y!, "MInt"); + if (x != null) { + __expect_type(x, "MInt"); + } + __expect_type(x, "MInt?"); + var (z1, z2) = (x, x!, ); + __expect_type(z1, "MInt?"); + __expect_type(z2, "MInt"); +} + +@method_id(104) +fun test4(x: MIntN, y: MIntN) { + if (x != null && y != null) { + __expect_type(x, "MInt"); + __expect_type(x + y, "int"); + return x + y; + } + __expect_type(x, "MInt?"); + __expect_type(x!, "MInt"); + __expect_type(rand() ? x : y, "MInt?"); + __expect_type(rand() ? x : 0, "MInt?"); + __expect_type(rand() ? 0 : x, "int?"); + __expect_type(rand() ? 0 : x!, "int"); + return y!; +} + +fun takeTensor_v1(v: Pair2_v1) { return v.0 + v.1; } +fun takeTensor_v2(v: Pair2_v2) { return v.0 + v.1; } +fun takeTensor_v3(v: (int, int)) { return v.0 + v.1; } + +fun test5() { + var x: Pair2_v1 = (1, 2); + var y = (3, 4) as Pair2_v2; + var z = (5, 6); + + __expect_type(x, "Pair2_v1"); + __expect_type(y, "Pair2_v2"); + + takeTensor_v1(x); takeTensor_v2(x); takeTensor_v3(x); + takeTensor_v1(y); takeTensor_v2(y); takeTensor_v3(y); + takeTensor_v1(z); takeTensor_v2(z); takeTensor_v3(z); + + var t = (y, x); + return t.0.1 + t.1.0; +} + +fun test6() { + var (x1: MInt?, x2: MIntN) = (5, 5); // smart cast + __expect_type(x1, "MInt"); + __expect_type(x2, "MInt"); + var (y1: MInt?, y2: MIntN) = (null, null); + __expect_type(y1, "null"); + __expect_type(y2, "null"); + var z: Pair2_v2? = (5, 5); + __expect_type(z, "Pair2_v2"); + __expect_type(z.0, "MInt"); +} + +fun someFn1(v: MInt): MIntN { return v; } + +fun test7() { + var f1: (int) -> int? = someFn1; + var f2: (int) -> MInt? = someFn1; + var f3: (MInt_v2) -> MInt_v2? = someFn1; + + f1 = f2; f1 = f3; + f2 = f1; f2 = f3; + f3 = f1; f3 = f2; +} + +fun test8() { + 0 as MInt; + 0 as MInt?; + 0 as MIntN; + + (1, 2) as Pair2_v2; + (1, 2) as Pair2_v2?; + (((1, 2) as Pair2_v2?) as (int, int)?) as Pair2_v1?; + + someFn1 as (int) -> int?; + someFn1 as (int) -> MInt?; + someFn1 as (MInt_v2) -> MInt_v2?; +} + +fun test9(b: MBool): MBool { + if (!b) { + __expect_type(b, "MBool"); + return !b; + } + return !!(5 as MInt); +} + +@method_id(110) +fun test10() { + var x1: Pair2_v1 = (5, 6); + var (a1, b1) = x1; + __expect_type(a1, "int"); + var x2: Pair2_v2? = x1; + var (a2, b2) = x2; + __expect_type(a2, "MInt"); + var x3: Tuple2Int = [9, 10]; + var [a3, b3, ] = x3; + return a1 + a2 + a3 + b1 + b2 + b3; +} + +fun analyzeTensor1(a: (T1, T2)): (T1, T2) { return a; } +fun analyzeTensor2(a: (T1, T2)?): (T1, T2)? { return a; } + +fun test11() { + var (x1: (int8, int16), x2: Pair2_v1, x3: Pair2_v2) = ((1,2), (3,4), (5,6)); + __expect_type(analyzeTensor1(x1), "(int8, int16)"); + __expect_type(analyzeTensor1(x2), "(int, int)"); + __expect_type(analyzeTensor1(x3), "(MInt, MInt)"); + __expect_type(analyzeTensor2(x1), "(int8, int16)?"); + __expect_type(analyzeTensor2(x2), "(int, int)?"); + __expect_type(analyzeTensor2(x3), "(MInt, MInt)?"); +} + +fun dict.getFakeDepth(self) { return 1; } +fun cell.getFakeDepth(self) { return 2; } + +@method_id(112) +fun test12(makeNotNull: bool) { + var d = createEmptyDict(); + if (makeNotNull) { + d.iDictSet(32, 123, ""); + } + var t = createEmptyTuple(); + if (d != null) { + __expect_type(d.getFakeDepth, "(cell) -> int"); + t.push(d.getFakeDepth()); + } else { + __expect_type(d.getFakeDepth, "(dict) -> int"); + t.push(d.getFakeDepth()); + } + t.push(d.getFakeDepth()); + return t; +} + + +fun main(x: MInt, y: MInt?) { + return y == null ? x : x + y; +} + +/** +@testcase | 0 | 3 4 | 7 +@testcase | 104 | 1 2 | 3 +@testcase | 110 | | 41 +@testcase | 112 | 0 | [ 1 1 ] +@testcase | 112 | -1 | [ 2 1 ] + +@fif_codegen +""" + test9() PROC:<{ // b + DUP // b b + IFNOTJMP:<{ // b + NOT // '2 + }> // b + DROP // + TRUE // '5 + }> +""" + */ diff --git a/tolk-tester/tests/unbalanced_ret_loops.tolk b/tolk-tester/tests/unbalanced-ret-loops.tolk similarity index 100% rename from tolk-tester/tests/unbalanced_ret_loops.tolk rename to tolk-tester/tests/unbalanced-ret-loops.tolk diff --git a/tolk-tester/tests/unbalanced-ret.tolk b/tolk-tester/tests/unbalanced-ret.tolk new file mode 100644 index 000000000..05c7f46d9 --- /dev/null +++ b/tolk-tester/tests/unbalanced-ret.tolk @@ -0,0 +1,73 @@ +fun main(x: int): (int, int) { + var y: int = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0) + } + } + return (x + 1, y) +} + +@inline +fun foo1(x: int): int { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} + +@method_id(101) +fun test1(x: int): int { + return foo1(x) * 10; +} + +fun foo2(y: int): int { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} + +fun bar2(x: int, y: int): (int, int) { + if (x < 0) { + y = foo2(y); + x *= 2; + if (x == -10) { + return (111, y) + } + } + return (x + 1, y); +} + +@method_id(102) +fun test2(x: int, y: int): (int, int) { + (x, y) = bar2(x, y); + return (x, y * 10); +} + +/** + method_id | in | out +@testcase | 0 | 10 | 11 5 +@testcase | 0 | -5 | 111 0 +@testcase | 0 | -4 | -7 6 +@testcase | 101 | 10 | 110 +@testcase | 101 | -5 | 1110 +@testcase | 101 | -4 | -70 +@testcase | 102 | 3 3 | 4 30 +@testcase | 102 | 3 -5 | 4 -50 +@testcase | 102 | 3 -4 | 4 -40 +@testcase | 102 | -5 3 | 111 40 +@testcase | 102 | -5 -5 | 111 1110 +@testcase | 102 | -5 -4 | 111 -70 +@testcase | 102 | -4 3 | -7 40 +@testcase | 102 | -4 -5 | -7 1110 +@testcase | 102 | -4 -4 | -7 -70 +*/ diff --git a/tolk-tester/tests/unbalanced_ret.tolk b/tolk-tester/tests/unbalanced_ret.tolk deleted file mode 100644 index 6cf42643a..000000000 --- a/tolk-tester/tests/unbalanced_ret.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun main(x: int): (int, int) { - var y: int = 5; - if (x < 0) { - x *= 2; - y += 1; - if (x == -10) { - return (111, 0); - } - } - return (x + 1, y); -} -/** - method_id | in | out -@testcase | 0 | 10 | 11 5 -@testcase | 0 | -5 | 111 0 -@testcase | 0 | -4 | -7 6 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_inline.tolk b/tolk-tester/tests/unbalanced_ret_inline.tolk deleted file mode 100644 index 4e24fbd8f..000000000 --- a/tolk-tester/tests/unbalanced_ret_inline.tolk +++ /dev/null @@ -1,19 +0,0 @@ -@inline -fun foo(x: int): int { - if (x < 0) { - x *= 2; - if (x == -10) { - return 111; - } - } - return x + 1; -} -fun main(x: int): int { - return foo(x) * 10; -} -/** - method_id | in | out -@testcase | 0 | 10 | 110 -@testcase | 0 | -5 | 1110 -@testcase | 0 | -4 | -70 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_nested.tolk b/tolk-tester/tests/unbalanced_ret_nested.tolk deleted file mode 100644 index 05e609240..000000000 --- a/tolk-tester/tests/unbalanced_ret_nested.tolk +++ /dev/null @@ -1,37 +0,0 @@ -fun foo(y: int): int { - if (y < 0) { - y *= 2; - if (y == -10) { - return 111; - } - } - return y + 1; -} -fun bar(x: int, y: int): (int, int) { - if (x < 0) { - y = foo(y); - x *= 2; - if (x == -10) { - return (111, y); - } - } - return (x + 1, y); -} -fun main(x: int, y: int): (int, int) { - (x, y) = bar(x, y); - return (x, y * 10); -} -/** - method_id | in | out -@testcase | 0 | 3 3 | 4 30 -@testcase | 0 | 3 -5 | 4 -50 -@testcase | 0 | 3 -4 | 4 -40 -@testcase | 0 | -5 3 | 111 40 -@testcase | 0 | -5 -5 | 111 1110 -@testcase | 0 | -5 -4 | 111 -70 -@testcase | 0 | -4 3 | -7 40 -@testcase | 0 | -4 -5 | -7 1110 -@testcase | 0 | -4 -4 | -7 -70 - -@code_hash 68625253347714662162648433047986779710161195298061582217368558479961252943991 -*/ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk new file mode 100644 index 000000000..7b9d52b61 --- /dev/null +++ b/tolk-tester/tests/union-types-tests.tolk @@ -0,0 +1,902 @@ +type MInt = int +type MNull = | null +type MSlice = slice +type MIntN = int | MNull +type IntOrSlice = MInt | slice +type IntOrSliceN = null | slice | int +type IntOrSliceOrBuilder = | int | slice | builder + +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); +type SomeIntBits = int8 | int9 | int10 | int32 | uint64; + +fun getSlice32(): slice { return beginCell().storeInt(32, 32).endCell().beginParse(); } +fun getIntOrSlice(): int | slice { return 5; } + +fun testFlatten() { + __expect_type(0 as (int | int), "int"); + __expect_type(0 as (int | int | slice | builder | slice), "int | slice | builder"); + __expect_type(0 as (int? | int), "int?"); + __expect_type(0 as (slice | int? | slice? | cell), "slice | int | null | cell"); + __expect_type(0 as (| int | (int | int)), "int"); + __expect_type(0 as (slice | (slice | (slice | int) | (| cell | slice?))), "slice | int | cell | null"); + + __expect_type(0 as (MInt | MNull), "MInt?"); + __expect_type(0 as (MInt | slice | int), "MInt | slice"); + __expect_type(0 as (MIntN), "MIntN"); + __expect_type(0 as (MIntN | null), "int?"); + __expect_type(0 as (MIntN | MNull | MSlice), "int | null | MSlice"); + __expect_type(0 as (MIntN | MNull? | MSlice), "int | null | MSlice"); + __expect_type(0 as (MInt | IntOrSliceOrBuilder | MSlice), "MInt | slice | builder"); + __expect_type((1, 2) as (Pair2 | Pair2Or3 | (slice, int?)), "Pair2 | (int, int, int) | (slice, int?)"); + + __expect_type(0 as (int | int8), "int | int8"); + __expect_type(0 as (MInt | int8), "MInt | int8"); + __expect_type((0 as MInt) as (int | int8), "int | int8"); + __expect_type((0 as MInt) as (MInt | int8), "MInt | int8"); + + __expect_type(0 as IntOrSlice, "IntOrSlice"); + __expect_type(0 as IntOrSlice?, "MInt | slice | null"); + __expect_type(0 as IntOrSliceN, "IntOrSliceN"); + __expect_type(0 as IntOrSliceN?, "null | slice | int"); + + __expect_type(0 as (int?)?, "int?"); + __expect_type(0 as (int?)? | (int)?, "int?"); + + __expect_type(0 is (|builder), "bool"); + __expect_type(0 !is builder, "bool"); + __expect_type(0!!is builder, "bool"); + + __expect_type(match (0) { int => [] }, "[]"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 }, "int"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 as int8 } as int|int8, "int | int8"); + __expect_type(match (0 as int?) { int => "", null => null }, "slice?"); + __expect_type(match (0 as int?) { + int => 10>3 ? match(0) { int => 0 } : match (10>3 ? 6 : null) { null => 0, int => { return; } }, + null => match(null) { null => 0 } + }, "int"); +} + +@method_id(101) +fun test1() { + var a = getIntOrSlice(); + var b: MInt | MSlice = getIntOrSlice(); + var c: IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + __expect_type(b, "MInt | MSlice"); + __expect_type(c, "IntOrSlice"); + return (a, b, c); +} + +@method_id(102) +fun test2() { + var a: int | slice = 4; + __expect_type(a, "int"); + a = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = getSlice32(); + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(103) +fun test3(case: int): | int | slice | null { + var a: int | slice | null = 4; + if (case == 0) { + a = getSlice32() + } else if (case == 1) { + a = 10 + } else { + a = null + } + return a; +} + +@method_id(104) +fun test4() { + return ( + 5 as int | int, + (5 as int8) as int8 | int8, + true as bool | bool, + 5 as int | int | (int | slice), + (5 as int | int | (int | slice)) as int | slice | null, + ) +} + +@method_id(105) +fun test5() { + var a: int | int | IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = 5; + return (a, a as IntOrSlice, a as IntOrSliceOrBuilder, a as MIntN); +} + +@method_id(106) +fun test6(case: int): int | slice | builder | null { + var a: int? | slice? | builder = null; + __expect_type(a, "null"); + if (case == 1) { + a = null as slice? | builder; + __expect_type(a, "null | slice | builder"); + __expect_type(a!, "slice | builder"); + return a + } + if (case == 2) { + __expect_type(a = 2 as MInt, "MInt"); + __expect_type(a, "int"); + a = 2; + __expect_type(a, "int"); + return a + 0 + } + if (case == 3) { + a = null as slice?; + __expect_type(a, "slice?"); + return a + } + a = null as MNull; + __expect_type(a, "null"); + return a; +} + +@method_id(107) +fun test7(case: int) { + var a = (2, 3); + var b: Pair2 = (4, 5); + var c: Pair2Or3 = case == 1 ? a : case == 2 ? b : (6, 7, 8); + __expect_type(c, "Pair2Or3"); + c as Pair2Or3; + c as Pair2Or3 | Pair2Or3; + c as (int, int, int) | Pair2; + c as (int, int) | (int, int, int); + return (c, c as (int, int)? | (int, int, int) | null); +} + +@method_id(108) +fun test8() { + return ( + (2, 3) as (int, int) | (int, int, int) | null, + null as (int, int) | null | (int, int, int), + (2, 3) as Pair2Or3 | null, + (2, 3) as null | Pair2Or3, + null as Pair2Or3? + ) +} + +@method_id(109) +fun test9() { + var c: null | Pair2Or3 = (6, 7, 8) as Pair2Or3?; + __expect_type(c, "null | Pair2 | (int, int, int)"); + __expect_type(c!, "Pair2 | (int, int, int)"); + c! as Pair2Or3; + c! as (int, int, int) | Pair2; + (c! as (int, int) | (int, int, int)) as Pair2Or3; + return (c, c == null, c = null, c == null, c = null as Pair2Or3?, c == null, c as Pair2? | (int, int, int)? | null); +} + +@method_id(110) +fun test10() { + var t = getSlice32() as (int, (int, int)?, (int?, int)?) | slice; + if (10 > 3) { + t = (1, null, null) as (int, (int, int)?, (int?, int)?) + } + return t; +} + +@method_id(120) +fun test20() { + var a = getIntOrSlice(); + if (a is int) { + __expect_type(a, "int"); + return a + 0; + } + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(121) +fun test21() { + var a = getIntOrSlice(); + if (a is int) { + if (a !is int8) { + __expect_type(a, "int"); + } + if (a is slice) { + __expect_type(a, "never"); + } + if (a !is slice) { + __expect_type(a, "int"); + } + if (a !is builder) { + __expect_type(a, "int"); + } + } + if (a !is int && a !is builder && a !is int && a !is null) { + __expect_type(a, "slice"); + } + if (a !is int || (a !is int && a !is builder)) { + __expect_type(a, "slice"); + } + if (a is int || a is slice) { + __expect_type(a, "int | slice"); + } else { + __expect_type(a, "never"); + } + if (a is slice || a is int) { + __expect_type(a, "slice | int") + } else { + __expect_type(a, "never") + } + if (a is int || a is slice || a is builder) { + __expect_type(a, "int | slice"); + } + if (a !is int && a !is slice && a !is builder) { + __expect_type(a, "never"); + } + return (a is int, a !is int, a is slice, a !is slice, a is int || a is slice || a is builder, a is builder); +} + +@method_id(122) +fun test22(a: int, b: int | slice) { + var result = 0; + if (a is int) { result += 1 } + if (a is int || a is slice) { result += 2 } + if (a is builder || a is slice) { result += 3 } + if (b is int) { result += 10 } + if ((b is int) | (b is slice)) { result += 11 } + if ((b is builder) | (b is int)) { result += 12 } + return result +} + +@method_id(123) +fun test23(p2: bool) { + var p: Pair2Or3 = p2 ? (1, 2) : (3, 4, 5); + __expect_type(p, "Pair2Or3"); + if (p is (int, int) && 10 < 0) { + __expect_type(p, "(int, int)"); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p is int) { + __expect_type(p, "never"); + return (0, 0); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p !is (int, int)) { + __expect_type(p, "(int, int, int)"); + p = (10, 20); + } + __expect_type(p, "Pair2"); + return p; +} + +fun eqUnion2(v: T1 | T2): T2 | T1 { return v } + +fun detectUnionSide(v: T1 | T2) { + if (v is T1) { + assert(v is T1 && v !is T2 && v !is (T1, T2), 101); + return 0; + } + assert(v is T2 && v !is T1 && !(v is null || v is never), 102); + return 1; +} + +@method_id(124) +fun test24() { + var bc = beginCell() as builder | cell; + var mi1 = 5 as MInt | int8; + var mi2 = 5 as int8 | MInt; + var mi3 = (5 as int8) as int8 | MInt; + __expect_type(eqUnion2(bc), "cell | builder"); + __expect_type(eqUnion2(mi1), "int8 | MInt"); + return ( + detectUnionSide(5), + detectUnionSide(5), + detectUnionSide(getIntOrSlice()), + detectUnionSide((getIntOrSlice() is int) ? getSlice32() : beginCell()), + detectUnionSide(bc), + detectUnionSide(mi1), + detectUnionSide(mi2), + detectUnionSide(mi3) + ); +} + +@method_id(125) +fun test25(a: int? | slice) { + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a is null) { result |= 0x0010; } + if (a is int || a is builder) { result |= 0x0020; } + if (a is int8 || a is null || a is int16) { result |= 0x0040; } + return result; +} + +@method_id(126) +fun test26() { + var a = 1 as int? | slice; + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a == null) { result |= 0x0010; } + if ((a is int) | a is builder) { result |= 0x0020; } + if ((a is int8) | (a is int16) | (a is null)) { result |= 0x0040; } + return result; +} + +@method_id(127) +fun test27() { + var a: (MInt, int) | slice = (1, 2); + __expect_type(a, "(MInt, int)"); + a = (1 as MInt, 2 as MInt); + __expect_type(a, "(MInt, int)"); + var b: IntOrSliceOrBuilder = 5 as (MInt | slice); + return (a, b); +} + +@method_id(128) +fun test28() { + var t = null as int | (int, int)?; + return (t, 777, 2 as int | (int, int)?, 777, (3, 4) as int | (int, int)?); +} + +@method_id(129) +fun test29() { + var t1 = (null as int?) as (int, int) | int | null; + var t2 = (5 as int?) as (int, int) | int | null; + return (t1, t2); +} + +fun get5OrNull(getNull: bool) { return getNull ? null : 5; } + +@method_id(130) +fun test30(getNull: bool) { + var t1 = get5OrNull(getNull) as (int, int) | int | null; + var t2 = (null as (int, int)?) as (int, int) | int | null; + var t3 = ((1, 2) as (int, int)?) as (int, int) | int | null; + var t4 = ([6] as [int]?) as (int, int) | [int] | null; + var t5 = (null as [int]?) as (int, int) | [int] | null; + return (t1, 777, t2, 777, t3, 777, t4, 777, t5); +} + +@method_id(131) +fun test31() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, int) | int | null = null; + var t2: (int, int) | int | null = 5; + var t3: (int, int) | int | null = (1, 2); + var t4: (int, int) | int | null = null as int?; + var t5: (int, int) | int | null = 5 as int?; + var t6: (int, int) | int | null = null as (int, int)?; + var t7: (int, int) | int | null = (1, 2) as (int, int)?; + { + __expect_type(t1, "null"); + __expect_type(t2, "int"); + __expect_type(t3, "(int, int)"); + __expect_type(t4, "int?"); + __expect_type(t6, "(int, int)?") + } + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(132) +fun test32() { + var t1 = 5 as (int, slice?, int) | (int, int) | int | int8 | null; + var t2 = (5 as int8) as (int, slice?, int) | (int, int) | int | int8 | null; + var t3 = (4, 5) as (int, slice?, int) | (int, int) | int | int8 | null; + var t4 = null as (int, slice?, int) | (int, int) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(133) +fun test33() { + var t1 = (5 as (int, MInt) | int) as (int, slice?, int) | (int, int) | MInt | int8 | null; + var t2 = ((5 as int8) as null | int8 | int) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t3 = ((4, 5) as (int, MInt) | MInt) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t4 = (null as MInt | int8 | null) as (int, slice?, int) | (int, MInt) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(134) +fun test34() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, cell, slice?, int) | (int, int) | int | int8 | null = 5 as (int, int) | int; + var t2: (int, cell, slice?, int) | (int, MInt) | int | int8 | null = 5 as (int, int) | MInt | int8; + var t3: (int, cell, slice?, int) | (int, int) | MInt | int8 | null = (1, 2) as (int, int) | MInt; + var t4: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (1, 2) as (int, int) | int | null; + var t5: (int, cell, slice?, int) | (int, int) | int | int8 | null = (5 as int8) as (int, MInt) | int | int8; + var t6: (int, cell, slice?, int) | (MInt, int) | MInt | int8 | null = (5 as int8) as int8 | int; + var t7: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (5 as int8) as int8 | MInt | null; + __expect_type(t1, "(int, int) | int"); + __expect_type(t7, "int | int8 | null"); + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(135) +fun test35() { + var t1 = 5 as int | (int, MInt) | null; + var t2 = (1, 2) as int | (int, int) | (int, MInt, slice, slice); + var t3 = (1, 2) as int | null | (MInt, int) | (MInt, int, slice, slice); + var t4 = null as int? | (MInt, MInt)?; + __expect_type(t1!, "int | (int, MInt)"); + __expect_type(t2!, "int | (int, int) | (int, MInt, slice, slice)"); + __expect_type(t3!, "int | (MInt, int) | (MInt, int, slice, slice)"); + __expect_type(t4!, "int | (MInt, MInt)"); + return (t1!, 777, t2!, 777, t3!, 777, t4!); +} + +@method_id(136) +fun test36() { + var t1 = (1 as int8, 2 as int16) as (int, int) | slice; + var t2 = (1 as int8, 2 as int16) as (int, MInt) | (int8, int16) | slice; + var t3 = (1 as int8, 2 as int16) as (int?, MInt?) | builder | null; + var t4 = (1 as int8, 2 as int16) as (int8?, int16) | (int, int, int) | (cell, cell, cell); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(137) +fun test37(one: int8, two: int16) { + var t1: (int, MInt) | slice = (one, two); + var t2: (int, int) | (int8, int16) | slice = (one, two); + var t3: (int?, int?) | builder | null = (one, two); + var t4: [int8?, int16] | (int, int, int) | (cell, cell, cell) = [one, two]; + __expect_type(t1, "(int, MInt)"); + __expect_type(t2, "(int8, int16)"); + __expect_type(t3, "(int?, int?)"); + __expect_type(t4, "[int8?, int16]"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(138) +fun test38() { + var t1 = (1, null, 2) as (int, (int, MInt, int, int)?, int) | slice; + var t2 = (1, null, 2) as (int, MInt | slice | null, int) | slice; + var t3 = (1, 5, 2) as (int, int | slice | null, int) | slice; + var t4 = (1, 5, 2) as (int, int | slice | (int, int), int) | slice; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(139) +fun test39() { + var t1: (int, (MInt, int, int, MInt)?, int) | slice = (1, null, 2); + var t2: (int, int | slice | null, int) | slice = (1, null, 2); + var t3: (int, MInt | slice | null, int) | slice = (1, 5, 2); + var t4: (int, int | slice | (int, int), int) | slice = (1, 5, 2); + __expect_type(t1, "(int, (MInt, int, int, MInt)?, int)"); + __expect_type(t3, "(int, MInt | slice | null, int)"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(140) +fun test40(x: int): SomeIntBits { + if (x < 10) { return x as int8 } + if (x < 20) { return x as int9 } + if (x < 30) { return x as int10 } + return x as uint64; +} + +@method_id(141) +fun test41(x: int?, y: MInt | MNull) { + var t1 = x as int | MNull; + var t2 = x as int | slice | MNull; + var t3 = y as int?; + var t4 = y as int? | slice; + return (t1, t2, t3, t4); +} + +@method_id(142) +fun test42(x: int?, y: MInt | MNull) { + var t1: int | MNull = x; + var t2: int | slice | MNull = x; + var t3: int? = y; + var t4: int? | slice = y; + __expect_type(t1, "int?"); + __expect_type(t2, "int?"); + __expect_type(t3, "int?"); + __expect_type(t4, "int?"); + return (t1, t2, t3, t4); +} + +@method_id(143) +fun test43() { + var a: MNull = null; + __expect_type(a, "null"); + return (a as null, a as int?, a as MIntN, a as (int, int)?, a as (int, int) | slice | MNull); +} + +@method_id(144) +fun test44(a: int) { + return (a is slice, a is int, a is null, a != null, a is builder, a !is (int, int), a is int8); +} + +@method_id(145) +fun test45(a: slice?) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int); +} + +@method_id(146) +fun test46(a: (int, int) | int) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int && a == 0); +} + +fun checkSimple(a: int | slice | builder) { + return match (a) { + int => 1, + slice => 2, + builder => 3, + } +} + +@method_id(147) +fun test47() { + __expect_type(checkSimple, "(int | slice | builder) -> int"); + return (checkSimple(1), checkSimple(beginCell().endCell().beginParse()), checkSimple(100), checkSimple(beginCell())); +} + +@noinline +fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { + var vv: T1? = match (a) { T1 => a, T2 => null, T3 => null }; + return match (a) { + T1 => (1, a), + T2 => (2, match(a) { T2 => a }), + T3 => (3, null), + } +} + +type IntOrInt8OrInt16 = int | int8 | int16; + +@method_id(148) +fun test48(a: IntOrInt8OrInt16) { + var b = a; + __expect_type(b, "IntOrInt8OrInt16"); + match (b) { MInt => {} int8 => {} int16 => {} } + __expect_type(b, "int | int8 | int16"); + match (b) { int => {}, int8 => { return ((0, null), (0, null)); }, int16 => "" }; + __expect_type(b, "int | int16"); + return (checkGeneric3(a), checkGeneric3(5 as int | slice | builder)); +} + +@method_id(149) +fun test49(a: slice? | int) { + if (a !is int) { + return match (a) { + slice => 1, + null => 2, + } + } + return a + 0 +} + +type VeryComplexType = int | (MInt, int)? | builder? | int; + +@method_id(150) +fun test50() { + var a = 5 as int | slice | null; + var b = 5 as VeryComplexType; + test49(a); + test49(match (a) { int => a, slice => a, null => a, }); + __expect_type(b, "VeryComplexType"); + match (b) { int => {} Pair2 => {} builder => {} null => {} } + __expect_type(b, "int | (int, int) | builder | null"); + __expect_type(b!, "int | (int, int) | builder"); + match (b) { int => 100, Pair2 => {} builder => return -2, null => "null" } + __expect_type(b, "int | (int, int) | null"); + if (b !is null) { + match (b) { int => {}, Pair2 => {} } + } else { + match (b) { null => {} } + } + match (b is null) { + bool => {} + } + match (b = 5 as VeryComplexType) { int => b, (int, int) => { return -1; } null => {} builder => return -2 } + __expect_type(b, "int?"); + b is int && b!is int && b!!is int; + if (match(b) { int => 1, null => null } !is int) { + return 100 + } + return (b! + test49(match (a) { int => a, slice => a, null => a } as int|slice?)); +} + +@method_id(151) +fun test51(init: int) { + var a = init as int | slice; + match(a){} + __expect_type(a, "int | slice"); + match (a) { + int => { + a = init; + match (a) {}; + __expect_type(a, "int"); + a = a > 10 ? beginCell().storeInt(init, 32).endCell().beginParse() : 10; + __expect_type(a, "int | slice"); + if (a is slice) { + a = a.loadInt(32) + } + } + slice => { + if (a !is int) { + a = a.loadInt(32) as int8; + } + } + } + __expect_type(a, "int"); + return a; +} + +@method_id(152) +fun test52(init: int): int | builder { + var m = get5OrNull(false); + if (init > 10 && init < 15) { + return init > 20 ? init : beginCell(); + } + return match (init as int?) { + int => init, + null => beginCell() + } +} + +fun inc53(mutate x: int) { + x += 1; + return x; +} + +@method_id(153) +fun test53(a: int | builder) { + match (var a = 5 as int | slice) { + int => {} + slice => { __expect_type(a, "slice"); var b: slice? = a; return b.loadInt(32); } + } + match (val a = getIntOrSlice()) { + int => a + 0, + slice => {} + } + __expect_type(a, "int | builder"); + var cc = match (var a = getIntOrSlice()) { + int => inc53(mutate a), + slice => { return 777; } + }; + __expect_type(cc, "int"); + match (var (a, (b, c)) = (10, (20, 30))) { + (int, Pair2) => {} + } + return cc +} + +@method_id(154) +fun test54(): int|slice { + match (var a = getIntOrSlice()) { + int => { + __expect_type(a, "int"); + a = beginCell().storeInt(100, 32).endCell().beginParse(); + __expect_type(a, "slice"); + if (a.remainingBitsCount() < 100) { a = a.loadInt(32); } + else { a = 0; } + __expect_type(a, "int"); + return a; + } + slice => throw 123 + } + return 777; +} + +@method_id(155) +fun test55() { + var a = getIntOrSlice() as int | slice | null; + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return (-10, 0), + null => throw 123 + }; + __expect_type(a, "int | slice"); + return (a, cc); +} + +@method_id(156) +fun test56(a: int8, b: int16, useSecond: bool) { + var r1: int8 | int16 = useSecond ? b: a; + var r2: int = useSecond ? b: a; + var r3: int8 | (int, int) | int16? = useSecond ? b : a; + var r4 = (useSecond ? b : a) as int; + var r5 = (useSecond ? b : a) as int8? | int16; + var r6: int8 | int16 = match (val c: int16 | int8 | int8 = useSecond ? b : a) { int8 => c, int16 => c }; + var r7 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c, int16 => c } as int; + var r8 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c as int, int16 => c as int }; + var r9: int = match (val c = (useSecond ? b : a) as IntOrInt8OrInt16) { int8 => c, int16 => c, int => c }; + return (r1, r2, r3, r4, r5, r6, r7, r8, r9); +} + +@method_id(157) +fun test57(r3v: int) { + var r1 = null as int | () | null; + var r2 = () as int | () | null; + var r3 = () as (int, int) | () | null; + var r4 = null as (int, int) | () | null; + if (r3v == 1) { + r3 = (1, 2); + } + return (r1, r2, 777, r3, 777, r4 = r3v == 1 ? () : null, r4, 777, r1 is int, r1 is (), r1 is null, r3 is int, r3 is (int,int), r3 is (), r3 == null, 777, r4 is (), r4 is null); +} + +@method_id(158) +fun test58() { + var tn = () as ()?; + var aln: null = (tn = null); + __expect_type(tn, "null"); + return ((()) as () | (int, int) | null, (() as () | null) as () | (int, int) | null, tn, tn as ()?, aln); +} + +fun alwaysThrows123(): never { throw 123; } + +@method_id(159) +fun test59(a: int | slice | builder | null) { + try { + var b = 0; + var c = match (a) { + int => b + a, + slice => { + b = 10; + throw 456; + } + builder => alwaysThrows123(), + null => { + alwaysThrows123(); + } + }; + __expect_type(c, "int"); + return b + c; + } catch (excCode) { + return excCode; + } +} + +fun takeIntReturnVoid(a: int) { } + +fun returnVoidOrNull(willBeVoid: bool) { + var (i: int?, _: builder) = (willBeVoid ? 10 : null, beginCell()); + + var res = i != null ? takeIntReturnVoid(i) : null; + __expect_type(res, "void?"); + return res; +} + +@method_id(160) +fun test60() { + return (returnVoidOrNull(true), returnVoidOrNull(false)); +} + +@method_id(161) +fun test61(i: int32 | int64 | int) { + match (i) { + int32 => { return 10; } + int64 => { return 20; } + int => { return 30; } + } +} + + + +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 +@testcase | 101 | | 5 1 5 1 5 1 +@testcase | 102 | | 32 +@testcase | 103 | 1 | 10 1 +@testcase | 103 | 2 | (null) 0 +@testcase | 104 | | 5 5 -1 5 1 5 1 +@testcase | 105 | | 5 5 1 5 1 5 +@testcase | 106 | 1 | (null) 0 +@testcase | 106 | 2 | 2 1 +@testcase | 107 | 1 | (null) 2 3 typeid-1 (null) 2 3 typeid-1 +@testcase | 107 | 3 | 6 7 8 typeid-2 6 7 8 typeid-2 +@testcase | 108 | | (null) 2 3 typeid-1 (null) (null) (null) 0 (null) 2 3 typeid-1 (null) 2 3 typeid-1 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 typeid-2 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-7 +@testcase | 120 | | 5 +@testcase | 121 | | -1 0 0 -1 -1 0 +@testcase | 122 | 0 0 1 | 36 +@testcase | 122 | 0 0 4 | 14 +@testcase | 123 | 1 | 1 2 +@testcase | 123 | 0 | 10 20 +@testcase | 124 | | 0 1 0 1 0 0 1 0 +@testcase | 125 | null 0 | 87 +@testcase | 125 | 1 1 | 42 +@testcase | 126 | | 42 +@testcase | 127 | | 1 2 5 1 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 typeid-1 +@testcase | 129 | | (null) (null) 0 (null) 5 1 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 typeid-1 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 typeid-1 777 1 2 typeid-1 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 0 +@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-12 777 1 2 typeid-13 777 (null) 1 2 typeid-14 +@testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-18 777 1 (null) 0 2 typeid-19 777 1 5 1 2 typeid-19 777 1 (null) 5 1 2 typeid-20 +@testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 +@testcase | 140 | 5 | 5 42 +@testcase | 140 | 15 | 15 typeid-3 +@testcase | 140 | 90 | 90 49 +@testcase | 141 | 7 8 | 7 7 1 8 8 1 +@testcase | 141 | null null | (null) (null) 0 (null) (null) 0 +@testcase | 142 | 7 8 | 7 7 8 8 +@testcase | 142 | null null | (null) (null) (null) (null) +@testcase | 143 | | (null) (null) (null) (null) (null) 0 (null) (null) 0 +@testcase | 144 | 0 | 0 -1 0 -1 0 -1 0 +@testcase | 145 | null | 0 -1 0 0 -1 0 +@testcase | 146 | null 0 1 | 0 0 -1 0 -1 -1 +@testcase | 147 | | 1 2 1 3 +@testcase | 148 | 9 1 | 1 9 1 1 5 1 +@testcase | 148 | 9 44 | 3 (null) 0 1 5 1 +@testcase | 149 | null 0 | 2 +@testcase | 150 | | 10 +@testcase | 151 | 88 | 88 +@testcase | 152 | 52 | 52 1 +@testcase | 153 | 1 1 | 6 +@testcase | 154 | | 100 1 +@testcase | 155 | | 5 1 5 1 +@testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 +@testcase | 157 | 1 | (null) 0 (null) typeid-21 777 1 2 typeid-1 777 typeid-21 typeid-21 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) typeid-21 777 (null) (null) typeid-21 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) typeid-21 (null) (null) typeid-21 (null) 0 (null) +@testcase | 159 | 0 4 | 456 +@testcase | 159 | null 0 | 123 +@testcase | 160 | | 10 0 +@testcase | 161 | 100 48 | 20 + + +@fif_codegen +""" + test26() PROC:<{ + 42 PUSHINT // result + }> +""" + +@fif_codegen +""" + checkSimple() PROC:<{ // a.USlot1 a.UTag + NIP // a.UTag + DUP // a.UTag a.UTag + 1 EQINT // a.UTag '3 + IF:<{ // a.UTag + DROP // + 1 PUSHINT // '2=1 + }>ELSE<{ // a.UTag + 4 EQINT // '6 + IF:<{ // + 2 PUSHINT // '2=2 + }>ELSE<{ // + 3 PUSHINT // '2=3 + }> + }> + }> +""" + +@fif_codegen +""" + test61() PROC:<{ + NIP + DUP + 46 EQINT + IFJMP:<{ + DROP + 10 PUSHINT + }> + 48 EQINT + IFJMP:<{ + 20 PUSHINT + }> + 30 PUSHINT + }> +""" + +@fif_codegen DECLPROC checkGeneric3() + + */ diff --git a/tolk-tester/tests/unreachable-5.tolk b/tolk-tester/tests/unreachable-5.tolk new file mode 100644 index 000000000..9371ee127 --- /dev/null +++ b/tolk-tester/tests/unreachable-5.tolk @@ -0,0 +1,24 @@ +fun main(x: int | slice | null) { + if (x is builder) { + __expect_type(x, "never"); + return 10; + } + if (x !is cell) { + __expect_type(x, "int | slice | null"); + } + if (x is int) { + return x; + } + return (x is null) ? -1 : 20; +} + +/** +@testcase | 0 | 0 1 | 0 +@testcase | 0 | null 0 | -1 +@testcase | 0 | -1 2 | 20 + +@stderr warning: variable `x` of type `int | slice | null` can never be `builder` +@stderr if (x is builder) +@stderr warning: variable `x` of type `int | slice | null` can never be `cell` +@stderr if (x !is cell) + */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 2a0e0e7ff..0c59c6715 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -9,7 +9,7 @@ fun main(): int { fun my_begin_cell(): builder asm "NEWC"; @pure -fun my_end_cell(b: builder): cell +fun builder.my_end_cell(self): cell asm "ENDC"; @pure fun my_begin_parse(c: cell): slice @@ -33,17 +33,30 @@ fun test1(): int { return demo_var + demo_slice; } +fun test2() { + return second; +} + global demo_slice: slice; const demo_20: int = 20; +const second = first + 1; +const first = 1; + /** @testcase | 0 | | 34 @fif_codegen """ - test1 PROC:<{ - // + test1() PROC:<{ 30 PUSHINT // '10 }> """ + +@fif_codegen +""" + test2() PROC:<{ + 2 PUSHINT // '2 + }> +""" */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk new file mode 100644 index 000000000..702c9fddc --- /dev/null +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -0,0 +1,371 @@ +type MInt = int; +type MIntN = int?; +type MBuilder = builder; +type CallbackIntToInt = int -> int; + +@pure +fun beginParse(c: cell): slice + asm "CTOS"; + +@pure +fun endCell(b: builder): cell + asm "ENDC"; + +fun getBeginCell() { + return beginCell +} + +fun getBeginParse() { + return beginParse +} + +@method_id(101) +fun testVarApply1() { + var (_, f_end_cell) = (0, endCell); + var b: MBuilder = (getBeginCell())().storeInt(1, 32); + b.storeInt(2, 32); + var s = (getBeginParse())(f_end_cell(b)); + return (s.loadInt(32), s.loadInt(32)); +} + +@inline +fun my_throw_always() { + throw 1000; +} + +@inline +fun get_raiser() { + return my_throw_always; +} + +@method_id(102) +fun testVarApplyWithoutSavingResult() { + try { + var raiser = get_raiser(); + raiser(); // `some_var()` is always impure, the compiler has no considerations about its runtime value + return 0; + } catch (code) { + return code; + } +} + +@inline +fun sum(a: int, b: int) { + assert(a + b < 24, 1000); + return a + b; +} + +@inline +fun mul(a: int, b: int) { + assert(a * b < 24, 1001); + return a * b; +} + +fun demo_handler(op: int, query_id: int, a: int, b: int): int { + if (op == 0xF2) { + val func = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return 0; // result not used, we test that func is nevertheless called + } + if (op == 0xF4) { + val func: (MInt, int) -> MInt = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return result; + } + return -1; +} + +@method_id(103) +fun testVarApplyInTernary() { + var t: tuple = createEmptyTuple(); + try { + t.push(demo_handler(0xF2, 122, 100, 200)); + } catch(code) { + t.push(code); + } + try { + t.push(demo_handler(0xF4, 122, 100, 200)) + } catch(code) { + t.push(code) + } + try { + t.push(demo_handler(0xF2, 122, 10, 10)) + } catch(code) { + t.push(code) + } + try { + t.push(demo_handler(0xF2, 123, 10, 10)) + } catch(code) { + t.push(code) + } + return t; +} + +fun always_throw2(x: int) { + throw 239 + x +} + +global global_f: MInt -> void; + +@method_id(104) +fun testGlobalVarApply() { + try { + global_f = always_throw2; + global_f(1); + return 0; + } catch (code) { + return code; + } +} + +@method_id(105) +fun testVarApply2() { + var creator = createEmptyTuple; + var t = creator(); + t.push(1); + var sizer = t.size; + return sizer(t); +} + +fun getTupleLastGetter(): (tuple) -> X { + return X.last; +} + +@method_id(106) +fun testVarApply3() { + var t = createEmptyTuple(); + t.push(1); + t.push([2]); + var getIntAt = t.get; + var getTupleFirstInt = createEmptyTuple().first; + var getTupleLastTuple = getTupleLastGetter(); + return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); +} + +@method_id(107) +fun testIndexedAccessApply() { + var functions1 = (beginCell, endCell); + var functions2 = [beginParse]; + var b = functions1.0().storeInt(1, 16); + b.storeInt(1, 16); + return functions2.0(functions1.1(b)).loadInt(32); +} + +fun getNullable4(): int? { return 4 } +fun myBeginCell(): builder? asm "NEWC"; + +@method_id(108) +fun testCallingNotNull() { + var n4: () -> MIntN = getNullable4; + var creator: (() -> builder?)? = myBeginCell; + var end2: [int, (MBuilder -> cell)?] = [0, endCell]; + var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); + return c.beginParse().loadInt(32); +} + +fun sumOfTensorIfNotNull(t: (int, int)?) { + if (t == null) { return 0; } + return t!.0 + t!.1; +} + +@method_id(109) +fun testTypeTransitionOfVarCall() { + var summer = sumOfTensorIfNotNull; + var hh1 = [1, null]; + var tt1 = (3, 4); + return (summer(null), summer((1,2)), summer(hh1.1), summer(tt1)); +} + +fun makeTensor(x1: int, x2: int, x3: int, x4: int, x5: int) { + return (x1, x2, x3, x4, x5); +} + +fun eq(x: T): T { return x; } + +@method_id(110) +fun testVarsModificationInsideVarCall(x: int) { + var cb = makeTensor; + return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; +} + +fun check_assoc_1(op: (int, int) -> int, a: int, b: int, c: int) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +@method_id(111) +fun testApplyNativePlus(x: int, y: int, z: int): bool { + return check_assoc_1(`_+_`, x, y, z); +} + +global op: (int, int) -> int; + +fun check_assoc_2(a: int, b: MInt, c: MIntN): bool { + return op(op(a, b), c!) == op(a, op(b, c!)); +} + +@method_id(112) +fun testApplyGlobalVar(x: int, y: int, z: int): bool? { + op = `_+_`; + if (0) { return null; } + return check_assoc_2(x, y, z); +} + +fun justAdd2(x: MInt): int { return x + 2; } + +@method_id(113) +fun testCallbackAlias(secondNull: bool) { + var adder1: CallbackIntToInt = justAdd2; + var adder2: CallbackIntToInt? = secondNull ? null : justAdd2; + return (adder1(10), adder2 == null ? -1 : adder2(10)); +} + +fun justGetTensor2() { return (2, 3) } + +@method_id(114) +fun testCallbackUnion(call1st: bool): (int, int) | int { + var cb: (int -> int) | (() -> (MInt, int)) = call1st ? justGetTensor2 : justAdd2; + if (cb is () -> (int, int)) { + return cb() + } + return cb(10) as int +} + +struct WithCallbacks { + f1: (int -> int)?, + beginCell: builder -> cell, +} + +fun WithCallbacks.doEndCellActually(self, b: builder) { return self.beginCell(b); } + +@method_id(115) +fun testCallableFields() { + var w: WithCallbacks = { f1: null, beginCell: endCell }; + if (10 > 3) { + w.f1 = justAdd2; + } + return ( + w.f1!(5), + w.doEndCellActually(beginCell().storeInt(6, 32)).beginParse().loadInt(32), + beginParse(w.beginCell(beginCell().storeInt(7, 32))).loadInt(32) + ); +} + +struct Wrapper { + value: T; +} + +fun wrap(value: T): Wrapper { + return{value}; +} + +@method_id(116) +fun testCallableGenericField() { + var w1: Wrapper<(int) -> int> = { value: justAdd2 }; + var w2: Wrapper = { value: justAdd2 }; + var w_f = wrapMInt>; + return (w1.value(5), w2.value!(9), wrap(w_f(justAdd2)).value.value(78)); +} + +struct Point { + x: int; + y: int; +} + +fun Point.createXY(x: int, y: int): Point { return {x,y}; } +fun Point.createFrom(x: U, y: V): Point { return {x,y}; } + +fun Point.maxCoord(self) { return self.x > self.y ? self.x : self.y; } + +@method_id(117) +fun testStaticAndInstanceMethods() { + var fPointCreateXY = Point.createXY; + var p1 = fPointCreateXY(5,6); + var fPointCreateFrom = Point.createFrom; + var p2 = fPointCreateFrom(7,8); + __expect_type(p1, "Point"); + __expect_type(p2, "Point"); + var getM1 = Point.maxCoord; + var getM2 = p1.maxCoord; + return (getM1(p1), (getM2 = p2.maxCoord)(p2)); +} + +fun Wrapper.createFromNull(): Wrapper { + return {value: null}; +} +fun Wrapper.createFrom(value: U): Wrapper { + return {value: value as T}; +} +fun Wrapper.equalsNotNull(self, anotherW: Wrapper) { + return self.value != null && anotherW.value != null && self.value == anotherW.value; +} + +@method_id(118) +fun testMethodsOfGenericStruct() { + var creator1 = Wrapper.createFrom; + var w1 = creator1(10); + __expect_type(w1, "Wrapper"); + var creator2 = Wrapper.createFrom; + var w2 = creator2(null as int16?); + __expect_type(w2, "Wrapper"); + var eq = Wrapper.equalsNotNull; + __expect_type(eq, "(Wrapper, Wrapper) -> bool"); + var creator3 = Wrapper.createFromNull; + __expect_type(creator3, "() -> Wrapper"); + return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); +} + +fun add3WithDefaults(a: int, b: int = 0, c: int = 0) { + return a + b + c; +} + +@method_id(119) +fun testSavingFunWithDefaults() { + var cc = add3WithDefaults; + __expect_type(cc, "(int, int, int) -> int"); + return cc(1, 2, 3); +} + +@noinline +fun tupleCreator() { return createEmptyTuple() } +@noinline +fun getTupleCreator() { return tupleCreator } + +@method_id(120) +fun testNoInlineSaving() { + var creator = getTupleCreator(); + __expect_type(creator, "() -> tuple"); + return creator(); +} + + +fun main() {} + +/** +@testcase | 101 | | 1 2 +@testcase | 102 | | 1000 +@testcase | 103 | | [ 1000 1000 0 1001 ] +@testcase | 104 | | 240 +@testcase | 105 | | 1 +@testcase | 106 | | 1 1 [ 2 ] [ 2 ] +@testcase | 107 | | 65537 +@testcase | 108 | | 4 +@testcase | 109 | | 0 3 0 7 +@testcase | 110 | 5 | 5 10 100 100 100 typeid-6 +@testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 +@testcase | 111 | 2 3 9 | -1 +@testcase | 111 | 11 22 44 | -1 +@testcase | 111 | -1 -10 -20 | -1 +@testcase | 112 | 2 3 9 | -1 +@testcase | 112 | 11 22 44 | -1 +@testcase | 112 | -1 -10 -20 | -1 +@testcase | 113 | 0 | 12 12 +@testcase | 113 | -1 | 12 -1 +@testcase | 114 | -1 | 2 3 typeid-3 +@testcase | 114 | 0 | (null) 12 1 +@testcase | 115 | | 7 6 7 +@testcase | 116 | | 7 11 80 +@testcase | 117 | | 6 8 +@testcase | 118 | | 0 10 -1 (null) +@testcase | 119 | | 6 +@testcase | 120 | | [] + */ diff --git a/tolk-tester/tests/var-apply.tolk b/tolk-tester/tests/var-apply.tolk deleted file mode 100644 index d189430fb..000000000 --- a/tolk-tester/tests/var-apply.tolk +++ /dev/null @@ -1,192 +0,0 @@ -fun getBeginCell() { - return beginCell; -} - -fun getBeginParse() { - return beginParse; -} - -@method_id(101) -fun testVarApply1() { - var (_, f_end_cell) = (0, endCell); - var b: builder = (getBeginCell())().storeInt(1, 32); - b.storeInt(2, 32); - var s = (getBeginParse())(f_end_cell(b)); - return (s.loadInt(32), s.loadInt(32)); -} - -@inline -fun my_throw_always() { - throw 1000; -} - -@inline -fun get_raiser() { - return my_throw_always; -} - -@method_id(102) -fun testVarApplyWithoutSavingResult() { - try { - var raiser = get_raiser(); - raiser(); // `some_var()` is always impure, the compiler has no considerations about its runtime value - return 0; - } catch (code) { - return code; - } -} - -@inline -fun sum(a: int, b: int) { - assert(a + b < 24, 1000); - return a + b; -} - -@inline -fun mul(a: int, b: int) { - assert(a * b < 24, 1001); - return a * b; -} - -fun demo_handler(op: int, query_id: int, a: int, b: int): int { - if (op == 0xF2) { - val func = query_id % 2 == 0 ? sum : mul; - val result = func(a, b); - return 0; // result not used, we test that func is nevertheless called - } - if (op == 0xF4) { - val func = query_id % 2 == 0 ? sum : mul; - val result = func(a, b); - return result; - } - return -1; -} - -@method_id(103) -fun testVarApplyInTernary() { - var t: tuple = createEmptyTuple(); - try { - t.tuplePush(demo_handler(0xF2, 122, 100, 200)); - } catch(code) { - t.tuplePush(code); - } - try { - t.tuplePush(demo_handler(0xF4, 122, 100, 200)); - } catch(code) { - t.tuplePush(code); - } - try { - t.tuplePush(demo_handler(0xF2, 122, 10, 10)); - } catch(code) { - t.tuplePush(code); - } - try { - t.tuplePush(demo_handler(0xF2, 123, 10, 10)); - } catch(code) { - t.tuplePush(code); - } - return t; -} - -fun always_throw2(x: int) { - throw 239 + x; -} - -global global_f: int -> void; - -@method_id(104) -fun testGlobalVarApply() { - try { - global_f = always_throw2; - global_f(1); - return 0; - } catch (code) { - return code; - } -} - -@method_id(105) -fun testVarApply2() { - var creator = createEmptyTuple; - var t = creator(); - t.tuplePush(1); - var sizer = t.tupleSize; - return sizer(t); -} - -fun getTupleLastGetter(): (tuple) -> X { - return tupleLast; -} - -@method_id(106) -fun testVarApply3() { - var t = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush([2]); - var getIntAt = t.tupleAt; - var getTupleFirstInt = createEmptyTuple().tupleFirst; - var getTupleLastTuple = getTupleLastGetter(); - return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); -} - -@method_id(107) -fun testIndexedAccessApply() { - var functions1 = (beginCell, endCell); - var functions2 = [beginParse]; - var b = functions1.0().storeInt(1, 16); - b.storeInt(1, 16); - return functions2.0(functions1.1(b)).loadInt(32); -} - -fun getNullable4(): int? { return 4; } -fun myBeginCell(): builder? asm "NEWC"; - -@method_id(108) -fun testCallingNotNull() { - var n4: () -> int? = getNullable4; - var creator: (() -> builder?)? = myBeginCell; - var end2: [int, (builder -> cell)?] = [0, endCell]; - var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); - return c.beginParse().loadInt(32); -} - -fun sumOfTensorIfNotNull(t: (int, int)?) { - if (t == null) { return 0; } - return t!.0 + t!.1; -} - -@method_id(109) -fun testTypeTransitionOfVarCall() { - var summer = sumOfTensorIfNotNull; - var hh1 = [1, null]; - var tt1 = (3, 4); - return (summer(null), summer((1,2)), summer(hh1.1), summer(tt1)); -} - -fun makeTensor(x1: int, x2: int, x3: int, x4: int, x5: int) { - return (x1, x2, x3, x4, x5); -} - -fun eq(x: T): T { return x; } - -@method_id(110) -fun testVarsModificationInsideVarCall(x: int) { - var cb = makeTensor; - return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; -} - -fun main() {} - -/** -@testcase | 101 | | 1 2 -@testcase | 102 | | 1000 -@testcase | 103 | | [ 1000 1000 0 1001 ] -@testcase | 104 | | 240 -@testcase | 105 | | 1 -@testcase | 106 | | 1 1 [ 2 ] [ 2 ] -@testcase | 107 | | 65537 -@testcase | 108 | | 4 -@testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 -1 -@testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 - */ diff --git a/tolk-tester/tests/w1.tolk b/tolk-tester/tests/w1.tolk deleted file mode 100644 index eb06bec67..000000000 --- a/tolk-tester/tests/w1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(id: int): (int, int) { - if (id > 0) { - if (id > 10) { - return (2 * id, 3 * id); - } - } - return (5, 6); -} -/** - method_id | in | out -@testcase | 0 | 0 | 5 6 -@testcase | 0 | 4 | 5 6 -@testcase | 0 | 11 | 22 33 -*/ diff --git a/tolk-tester/tests/w6.tolk b/tolk-tester/tests/w6.tolk deleted file mode 100644 index 489ffa8cc..000000000 --- a/tolk-tester/tests/w6.tolk +++ /dev/null @@ -1,19 +0,0 @@ -fun main(x: int): int { - var i: int = 0; - // int f = false; - do { - i = i + 1; - if (i > 5) { - return 1; - } - var f: bool = (i * i == 64); - } while (!f); - return -1; -} - -/** - method_id | in | out -@testcase | 0 | 0 | 1 - -@code_hash 36599880583276393028571473830850694081778552118303309411432666239740650614479 -*/ diff --git a/tolk-tester/tests/w7.tolk b/tolk-tester/tests/w7.tolk deleted file mode 100644 index 3d68c775b..000000000 --- a/tolk-tester/tests/w7.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@method_id(1) -fun test(y: int): int { - var x: int = 1; - if (y > 0) { - return 1; - } - return x > 0 ? -1 : 0; -} - -@method_id(2) -fun f(y: int): int { - if (y > 0) { - return 1; - } - return 2; -} - -fun main() { } - -/** - method_id | in | out -@testcase | 1 | 10 | 1 -@testcase | 1 | -5 | -1 -@testcase | 2 | 10 | 1 -@testcase | 2 | -5 | 2 -*/ diff --git a/tolk-tester/tests/w9.tolk b/tolk-tester/tests/w9.tolk deleted file mode 100644 index b88dc736e..000000000 --- a/tolk-tester/tests/w9.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(s: int) { - var (z, t) = (17, s); - while (z > 0) { - t = s; - z -= 1; - } - return ~ t; -} - -/** - method_id | in | out -@testcase | 0 | 1 | -2 -@testcase | 0 | 5 | -6 -*/ diff --git a/tolk-tester/tests/warnings-1.tolk b/tolk-tester/tests/warnings-1.tolk deleted file mode 100644 index 040057d17..000000000 --- a/tolk-tester/tests/warnings-1.tolk +++ /dev/null @@ -1,28 +0,0 @@ -fun getNullableInt(): int? { return null; } - -fun main() { - var c: int? = 6; - __expect_type(c, "int"); - if (c == null) {} - - var d: int? = c; - if (((d)) != null && tupleSize(createEmptyTuple())) {} - - var e: int? = getNullableInt(); - if (e != null) { - return true; - } - __expect_type(e, "null"); - null == e; - - return null != null; -} - -/** -@testcase | 0 | | 0 - -@stderr warning: variable `c` of type `int` is always not null, this condition is always false -@stderr warning: variable `d` of type `int` is always not null, this condition is always true -@stderr warning: variable `e` is always null, this condition is always true -@stderr warning: expression is always null, this condition is always false - */ diff --git a/tolk-tester/tests/unreachable-1.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-1.tolk similarity index 100% rename from tolk-tester/tests/unreachable-1.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-1.tolk diff --git a/tolk-tester/tests/unreachable-2.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-2.tolk similarity index 100% rename from tolk-tester/tests/unreachable-2.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-2.tolk diff --git a/tolk-tester/tests/unreachable-3.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk similarity index 77% rename from tolk-tester/tests/unreachable-3.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-3.tolk index fab21fd24..14bc43264 100644 --- a/tolk-tester/tests/unreachable-3.tolk +++ b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk @@ -15,7 +15,7 @@ fun main(x: int?) { @testcase | 0 | 5 | -2 @testcase | 0 | null | -1 -@stderr warning: variable `x` of type `int` is always not null +@stderr warning: variable `x` of type `int` can never be `null`, this condition is always true @stderr if (x != null) @stderr warning: unreachable code @stderr return 3 + 4 diff --git a/tolk-tester/tests/unreachable-4.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-4.tolk similarity index 100% rename from tolk-tester/tests/unreachable-4.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-4.tolk diff --git a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk new file mode 100644 index 000000000..ec77a2a66 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk @@ -0,0 +1,28 @@ +fun getNullableInt(): int? { return null; } + +fun main() { + var c: int? = 6; + __expect_type(c, "int"); + if (c == null) {} + + var d: int? = c; + if (((d)) != null && createEmptyTuple().size()) {} + + var e: int? = getNullableInt(); + if (e != null) { + return true; + } + __expect_type(e, "null"); + null == e; + + return null != null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `c` of type `int` can never be `null`, this condition is always false +@stderr warning: variable `d` of type `int` can never be `null`, this condition is always true +@stderr warning: variable `e` is always `null`, this condition is always true +@stderr warning: expression is always `null`, this condition is always false + */ diff --git a/tolk-tester/tests/warnings-2.tolk b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk similarity index 63% rename from tolk-tester/tests/warnings-2.tolk rename to tolk-tester/tests/warnings-not-errors/warnings-2.tolk index 57ecb21ac..2c5d97e84 100644 --- a/tolk-tester/tests/warnings-2.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk @@ -15,12 +15,12 @@ fun main() { /** @testcase | 0 | | -1 -@stderr warning: variable `a` of type `int` is always not null, this condition is always true +@stderr warning: variable `a` of type `int` can never be `null`, this condition is always true @stderr warning: condition of ternary operator is always true -@stderr warning: variable `c` of type `slice` is always not null, this condition is always false +@stderr warning: variable `c` of type `slice` can never be `null`, this condition is always false @stderr warning: condition of `if` is always true -@stderr warning: variable `b` of type `builder` is always not null, this condition is always false +@stderr warning: variable `b` of type `builder` can never be `null`, this condition is always false @stderr warning: condition of `assert` is always false @stderr warning: condition of `while` is always false -@stderr warning: expression of type `bool` is always not null, this condition is always true +@stderr warning: expression of type `bool` can never be `null`, this condition is always true */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-3.tolk b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk new file mode 100644 index 000000000..1198bc104 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk @@ -0,0 +1,26 @@ +fun main() { + var x = 0 as int | slice; + if (x !is builder) {} + while ((x = x) is builder) {} + + match (val tt = x) { + int => { + assert (tt is int, 505); + } + slice => {} + } + assert(x is int) throw 400; + __expect_type(x, "int"); + return x == null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `x` of type `int | slice` can never be `builder`, this condition is always true +@stderr warning: condition of `if` is always true +@stderr warning: expression of type `int | slice` can never be `builder`, this condition is always false +@stderr warning: condition of `while` is always false +@stderr warning: variable `tt` is always `int`, this condition is always true +@stderr warning: condition of `assert` is always true + */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-4.tolk b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk new file mode 100644 index 000000000..bc843ecc7 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk @@ -0,0 +1,15 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w = Wrapper { value: 16 as int16 }; + return (w is Wrapper, w is Wrapper, w is Wrapper, w !is Wrapper< Wrapper >); +} + +/** +@testcase | 0 | | -1 -1 0 -1 + +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper`, this condition is always false +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper>`, this condition is always true + */ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index c7e710214..dcd667e6e 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -1,4 +1,4 @@ -// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js test_file.tolk` +// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js tests_dir file_pattern` // from current dir, providing some env (see getenv() calls). // This is a JS version of tolk-tester.py to test Tolk compiled to WASM. // Don't forget to keep it identical to Python version! @@ -32,32 +32,38 @@ const TMP_DIR = os.tmpdir() class CmdLineOptions { constructor(/**string[]*/ argv) { - if (argv.length !== 3) { - print("Usage: node tolk-tester.js tests_dir OR node tolk-tester.js test_file.tolk") + if (argv.length < 3) { + print("Usage: node tolk-tester.js tests_dir [file_pattern]") process.exit(1) } - if (!fs.existsSync(argv[2])) { - print(`Input '${argv[2]}' doesn't exist`) + if (!fs.existsSync(argv[2]) || !fs.lstatSync(argv[2]).isDirectory()) { + print(`Directory '${argv[2]}' doesn't exist`) process.exit(1) } - if (fs.lstatSync(argv[2]).isDirectory()) { - this.tests_dir = argv[2] - this.test_file = null - } else { - this.tests_dir = path.dirname(argv[2]) - this.test_file = argv[2] - } + this.tests_dir = argv[2] + this.file_pattern = argv[3] } /** @return {string[]} */ find_tests() { - if (this.test_file) // an option to run (debug) a single test - return [this.test_file] + let all_test_files = [] + let all_children_of_tests_dir = fs.readdirSync(this.tests_dir) + all_children_of_tests_dir.sort() + for (let f of all_children_of_tests_dir) + if (f.endsWith(".tolk")) + all_test_files.push(path.join(this.tests_dir, f)) + for (let f of all_children_of_tests_dir) + if (!f.endsWith(".tolk") && f !== "imports") { + let subdir = path.join(this.tests_dir, f) + let children_of_subdir = fs.readdirSync(subdir) + children_of_subdir.sort() + all_test_files.push(...children_of_subdir.map(f => path.join(subdir, f))) + } - let tests = fs.readdirSync(this.tests_dir).filter(f => f.endsWith('.tolk') || f.endsWith('.ton')) - tests.sort() - return tests.map(f => path.join(this.tests_dir, f)) + if (this.file_pattern) + all_test_files = all_test_files.filter(f => f.includes(this.file_pattern)) + return all_test_files } } @@ -124,9 +130,12 @@ class TolkTestCaseInputOutput { this.expected_output = output_str } - check(/**string[]*/ stdout_lines, /**number*/ line_idx) { - if (stdout_lines[line_idx] !== this.expected_output) - throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}): expected '${this.expected_output}', found '${stdout_lines[line_idx]}'`, stdout_lines.join("\n")) + check(/**string[]*/ stdout_lines, /**number*/ line_idx, /**number*/ pivot_typeid) { + let expected_str = this.expected_output + if (expected_str.includes("typeid")) + expected_str = expected_str.replace(/typeid-(\d+)/g, (match, p1) => pivot_typeid + (+p1)) + if (stdout_lines[line_idx] !== expected_str) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${expected_str}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) } } @@ -267,6 +276,10 @@ class TolkTestFile { this.expected_hash = null /** @type {string | null} */ this.experimental_options = null + /** @type {boolean} */ + this.enable_tolk_lines_comments = false + /** @type {number} */ + this.pivot_typeid = 138 // may be changed when stdlib introduces new union types } parse_input_from_tolk_file() { @@ -286,6 +299,8 @@ class TolkTestFile { this.stderr_includes.push(new TolkTestCaseStderr(this.parse_string_value(lines), false)) } else if (line.startsWith("@fif_codegen_avoid")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen_enable_comments")) { + this.enable_tolk_lines_comments = true } else if (line.startsWith("@fif_codegen")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), false)) } else if (line.startsWith("@code_hash")) { @@ -339,9 +354,9 @@ class TolkTestFile { async run_and_check() { const wasmModule = await compileWasm(TOLKFIFTLIB_MODULE, TOLKFIFTLIB_WASM) - let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options) + let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options, this.enable_tolk_lines_comments) let exit_code = res.status === 'ok' ? 0 : 1 - let stderr = res.message + let stderr = res.message || res.stderr let stdout = '' if (exit_code === 0 && this.compilation_should_fail) @@ -384,7 +399,7 @@ class TolkTestFile { throw new CompareOutputError(`unexpected number of fift output: ${stdout_lines.length} lines, but ${this.input_output.length} testcases`, stdout) for (let i = 0; i < stdout_lines.length; ++i) - this.input_output[i].check(stdout_lines, i) + this.input_output[i].check(stdout_lines, i, this.pivot_typeid) if (this.fif_codegen.length) { const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) @@ -400,7 +415,7 @@ class TolkTestFile { async function run_all_tests(/**string[]*/ tests) { for (let ti = 0; ti < tests.length; ++ti) { let tolk_filename = tests[ti] - print(`Running test ${ti + 1}/${tests.length}: ${tolk_filename}`) + print(`Running test ${ti + 1}/${tests.length}: ${path.basename(tolk_filename)}`) let artifacts_folder = path.join(TMP_DIR, tolk_filename) let testcase = new TolkTestFile(tolk_filename, artifacts_folder) @@ -413,7 +428,7 @@ async function run_all_tests(/**string[]*/ tests) { fs.rmSync(artifacts_folder, {recursive: true}) if (testcase.compilation_should_fail) - print(" OK, compilation failed as it should") + print(" OK, stderr match") else print(` OK, ${testcase.input_output.length} cases`) } catch (e) { @@ -482,7 +497,7 @@ function copyFromCString(mod, ptr) { } /** @return {{status: string, message: string, fiftCode: string, codeBoc: string, codeHashHex: string}} */ -function compileFile(mod, filename, experimentalOptions) { +function compileFile(mod, filename, experimentalOptions, withSrcLineComments) { // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { if (kind === 0) { // realpath @@ -514,6 +529,7 @@ function compileFile(mod, filename, experimentalOptions) { const config = { optimizationLevel: 2, withStackComments: true, + withSrcLineComments: withSrcLineComments, experimentalOptions: experimentalOptions || undefined, entrypointFileName: filename }; diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 0b3c774ca..da81f87fe 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -1,4 +1,4 @@ -# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py test_file.tolk` +# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py tests_dir file_pattern` # from current dir, providing some env (see getenv() calls). # Every .tolk file should provide /* testcase description in a comment */, consider tests/ folder. # @@ -36,27 +36,33 @@ def getenv(name, default=None): class CmdLineOptions: def __init__(self, argv: List[str]): - if len(argv) != 2: - print("Usage: tolk-tester.py tests_dir OR tolk-tester.py test_file.tolk", file=sys.stderr) + if len(argv) < 2: + print("Usage: tolk-tester.py tests_dir [file_pattern]", file=sys.stderr) exit(1) - if not os.path.exists(argv[1]): - print("Input '%s' doesn't exist" % argv[1], file=sys.stderr) + if not os.path.isdir(argv[1]): + print("Directory '%s' doesn't exist" % argv[1], file=sys.stderr) exit(1) - if os.path.isdir(argv[1]): - self.tests_dir = argv[1] - self.test_file = None - else: - self.tests_dir = os.path.dirname(argv[1]) - self.test_file = argv[1] + self.tests_dir = argv[1] + self.file_pattern = argv[2] if len(argv) > 2 else None def find_tests(self) -> List[str]: - if self.test_file is not None: # an option to run (debug) a single test - return [self.test_file] - - tests = [f for f in os.listdir(self.tests_dir) if f.endswith(".tolk") or f.endswith(".ton")] - tests.sort() - return [os.path.join(self.tests_dir, f) for f in tests] + all_test_files: List[str] = [] + all_children_of_tests_dir = os.listdir(self.tests_dir) + all_children_of_tests_dir.sort() + for f in all_children_of_tests_dir: + if f.endswith(".tolk"): + all_test_files.append(os.path.join(self.tests_dir, f)) + for f in all_children_of_tests_dir: + if not f.endswith(".tolk") and f != "imports": + subdir = os.path.join(self.tests_dir, f) + children_of_subdir = os.listdir(subdir) + children_of_subdir.sort() + all_test_files += [os.path.join(subdir, f) for f in children_of_subdir] + + if self.file_pattern is not None: + all_test_files = [f for f in all_test_files if f.find(self.file_pattern) != -1] + return all_test_files class ParseInputError(Exception): @@ -101,6 +107,7 @@ class TolkTestCaseInputOutput: """ reJustNumber = re.compile(r"[-+]?\d+") reMathExpr = re.compile(r"[0x123456789()+\-*/<>]+") + reGasUsed = re.compile(r"gas:\sused=(\d+)") def __init__(self, method_id_str: str, input_str: str, output_str: str): processed_inputs = [] @@ -109,6 +116,8 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): continue elif in_arg.startswith("x{") or TolkTestCaseInputOutput.reJustNumber.fullmatch(in_arg): processed_inputs.append(in_arg) + elif in_arg.startswith("cell{"): + processed_inputs.append("") elif TolkTestCaseInputOutput.reMathExpr.fullmatch(in_arg): processed_inputs.append(str(eval(in_arg))) elif in_arg == "null": @@ -120,9 +129,12 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): self.input = " ".join(processed_inputs) self.expected_output = output_str - def check(self, stdout_lines: List[str], line_idx: int): - if stdout_lines[line_idx] != self.expected_output: - raise CompareOutputError("error on case #%d (%d | %s): expected '%s', found '%s'" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + def check(self, stdout_lines: List[str], line_idx: int, pivot_typeid: int): + expected_str = self.expected_output + if expected_str.find("typeid") != -1: + expected_str = re.sub(r'typeid-(\d+)', lambda m: str(pivot_typeid + int(m.group(1))), expected_str) + if stdout_lines[line_idx] != expected_str: + raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, expected_str, stdout_lines[line_idx]), "\n".join(stdout_lines)) class TolkTestCaseStderr: @@ -253,6 +265,8 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.fif_codegen: List[TolkTestCaseFifCodegen] = [] self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None + self.enable_tolk_lines_comments = False + self.pivot_typeid = 138 # may be changed when stdlib introduces new union types def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: @@ -272,6 +286,8 @@ def parse_input_from_tolk_file(self): self.stderr_includes.append(TolkTestCaseStderr(self.parse_string_value(lines), False)) elif line.startswith("@fif_codegen_avoid"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen_enable_comments"): + self.enable_tolk_lines_comments = True elif line.startswith("@fif_codegen"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), False)) elif line.startswith("@code_hash"): @@ -319,6 +335,8 @@ def run_and_check(self): cmd_args = [TOLK_EXECUTABLE, "-o", self.get_compiled_fif_filename()] if self.experimental_options: cmd_args = cmd_args + ["-x", self.experimental_options] + if not self.enable_tolk_lines_comments: + cmd_args = cmd_args + ["-L"] res = subprocess.run(cmd_args + [self.tolk_filename], capture_output=True, timeout=10) exit_code = res.returncode stderr = str(res.stderr, "utf-8") @@ -331,7 +349,7 @@ def run_and_check(self): should_include.check(stderr) if exit_code != 0 and self.compilation_should_fail: - return + return 0 if exit_code != 0 and not self.compilation_should_fail: raise TolkCompilationFailedError("tolk exit_code = %d" % exit_code, stderr) @@ -350,6 +368,7 @@ def run_and_check(self): if exit_code != 0: raise FiftExecutionFailedError("fift exit_code = %d" % exit_code, stderr) + gas_used = sum(map(int, TolkTestCaseInputOutput.reGasUsed.findall(stderr))) stdout_lines = [x.strip() for x in stdout.split("\n")] stdout_lines = [x for x in stdout_lines if x != ""] fif_code_hash = None @@ -361,7 +380,7 @@ def run_and_check(self): raise CompareOutputError("unexpected number of fift output: %d lines, but %d testcases" % (len(stdout_lines), len(self.input_output)), stdout) for i in range(len(stdout_lines)): - self.input_output[i].check(stdout_lines, i) + self.input_output[i].check(stdout_lines, i, self.pivot_typeid) if len(self.fif_codegen): with open(self.get_compiled_fif_filename()) as fd: @@ -372,11 +391,14 @@ def run_and_check(self): if self.expected_hash is not None: self.expected_hash.check(fif_code_hash) + return gas_used + def run_all_tests(tests: List[str]): + total_gas_used = 0 for ti in range(len(tests)): tolk_filename = tests[ti] - print("Running test %d/%d: %s" % (ti + 1, len(tests), tolk_filename), file=sys.stderr) + print("Running test %d/%d: %s" % (ti + 1, len(tests), os.path.basename(tolk_filename)), file=sys.stderr) artifacts_folder = os.path.join(TMP_DIR, tolk_filename) testcase = TolkTestFile(tolk_filename, artifacts_folder) @@ -384,13 +406,14 @@ def run_all_tests(tests: List[str]): if not os.path.exists(artifacts_folder): os.makedirs(artifacts_folder) testcase.parse_input_from_tolk_file() - testcase.run_and_check() + gas_used = testcase.run_and_check() shutil.rmtree(artifacts_folder) + total_gas_used += gas_used if testcase.compilation_should_fail: - print(" OK, compilation failed as it should", file=sys.stderr) + print(" OK, stderr match", file=sys.stderr) else: - print(" OK, %d cases" % len(testcase.input_output), file=sys.stderr) + print(" OK, %d cases" % (len(testcase.input_output)), file=sys.stderr) except ParseInputError as e: print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr) exit(2) @@ -423,9 +446,10 @@ def run_all_tests(tests: List[str]): print(" Mismatch in code hash:", e, file=sys.stderr) print(" Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr) exit(2) + return total_gas_used tests = CmdLineOptions(sys.argv).find_tests() print("Found", len(tests), "tests", file=sys.stderr) -run_all_tests(tests) -print("Done, %d tests" % len(tests), file=sys.stderr) +total_gas_used = run_all_tests(tests) +print("Done, %d tests, gas %d" % (len(tests), total_gas_used), file=sys.stderr) diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index de4081157..033456e7f 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TOLK_SOURCE src-file.cpp lexer.cpp @@ -8,9 +6,14 @@ set(TOLK_SOURCE ast.cpp ast-from-tokens.cpp constant-evaluator.cpp + gen-entrypoints.cpp + pack-unpack-api.cpp + pack-unpack-serializers.cpp + send-message-api.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp + pipe-resolve-types.cpp pipe-calc-rvalue-lvalue.cpp pipe-infer-types-and-calls.cpp pipe-check-inferred-types.cpp @@ -19,12 +22,17 @@ set(TOLK_SOURCE pipe-check-pure-impure.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp + pipe-detect-inline-in-place.cpp + pipe-check-serialized-fields.cpp + pipe-lazy-load-insertions.cpp + pipe-transform-on-message.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp type-system.cpp smart-casts-cfg.cpp generics-helpers.cpp + lazy-helpers.cpp abscode.cpp analyzer.cpp asmops.cpp diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index fc1609846..462aad7b2 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -59,9 +59,6 @@ void VarDescr::show_value(std::ostream& os) const { if (val & _Int) { os << 'i'; } - if (val & _Const) { - os << 'c'; - } if (val & _Zero) { os << '0'; } @@ -114,7 +111,7 @@ void VarDescr::set_const(td::RefInt256 value) { if (!int_const->signed_fits_bits(257)) { int_const.write().invalidate(); } - val = _Const | _Int; + val = _Int; int s = sgn(int_const); if (s < -1) { val |= _Nan | _NonZero; @@ -130,41 +127,41 @@ void VarDescr::set_const(td::RefInt256 value) { } } -void VarDescr::set_const(std::string value) { - str_const = value; - val = _Const; +void VarDescr::set_const(const std::string&) { + int_const.clear(); + val = 0; } void VarDescr::operator|=(const VarDescr& y) { - val &= y.val; - if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { - val &= ~_Const; - } - if (!(val & _Const)) { - int_const.clear(); + if (is_int_const()) { + bool y_same = y.is_int_const() && *int_const == *y.int_const; + if (!y_same) { + int_const.clear(); + } } + val &= y.val; } void VarDescr::operator&=(const VarDescr& y) { - val |= y.val; - if (y.int_const.not_null() && int_const.is_null()) { + if (y.is_int_const()) { int_const = y.int_const; } + val |= y.val; } void VarDescr::set_value(const VarDescr& y) { - val = y.val; int_const = y.int_const; + val = y.val; } void VarDescr::set_value(VarDescr&& y) { - val = y.val; int_const = std::move(y.int_const); + val = y.val; } void VarDescr::clear_value() { - val = 0; int_const.clear(); + val = 0; } void VarDescrList::show(std::ostream& os) const { @@ -205,9 +202,6 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx dis += " "; } switch (cl) { - case _Undef: - os << pfx << dis << "???\n"; - break; case _Nop: os << pfx << dis << "NOP\n"; break; @@ -404,16 +398,32 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::vector ir_idx; int stack_w = var_type->get_width_on_stack(); ir_idx.reserve(stack_w); - if (const TypeDataTensor* t_tensor = var_type->try_as()) { + if (const TypeDataStruct* t_struct = var_type->try_as()) { + for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field(i); + std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name; + std::vector nested = create_var(field_ref->declared_type, loc, std::move(sub_name)); + ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); + } + } else if (const TypeDataTensor* t_tensor = var_type->try_as()) { for (int i = 0; i < t_tensor->size(); ++i) { std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); } - } else if (const TypeDataNullable* t_nullable = var_type->try_as(); t_nullable && stack_w != 1) { - std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; - ir_idx = create_var(t_nullable->inner, loc, std::move(name)); - ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); + } else if (const TypeDataAlias* t_alias = var_type->try_as()) { + ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); + } else if (const TypeDataUnion* t_union = var_type->try_as(); t_union && stack_w != 1) { + std::string utag_name = name.empty() ? "'UTag" : name + ".UTag"; + if (t_union->or_null) { // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag" + ir_idx = create_var(t_union->or_null, loc, std::move(name)); + } else { // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag" + for (int i = 0; i < stack_w - 1; ++i) { + std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1); + ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]); + } + } + ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); @@ -426,4 +436,15 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s return ir_idx; } +var_idx_t CodeBlob::create_int(SrcLocation loc, int64_t value, const char* desc) { + vars.emplace_back(var_cnt, TypeDataInt::create(), std::string{}, loc); +#ifdef TOLK_DEBUG + vars.back().desc = desc; +#endif + var_idx_t ir_int = var_cnt; + var_cnt++; + emplace_back(loc, Op::_IntConst, std::vector{ir_int}, td::make_refint(value)); + return ir_int; +} + } // namespace tolk diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index c38b0bfa4..8e9fe914e 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -62,10 +62,10 @@ bool operator==(const VarDescrList& x, const VarDescrList& y) { } bool same_values(const VarDescr& x, const VarDescr& y) { - if (x.val != y.val || x.int_const.is_null() != y.int_const.is_null()) { + if (x.val != y.val || x.is_int_const() != y.is_int_const()) { return false; } - if (x.int_const.not_null() && cmp(x.int_const, y.int_const) != 0) { + if (x.is_int_const() && *x.int_const != *y.int_const) { return false; } return true; @@ -501,7 +501,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } default: std::cerr << "fatal: unknown operation in compute_used_vars()\n"; - throw ParseError{where, "unknown operation"}; + throw ParseError(loc, "unknown operation"); } } @@ -583,7 +583,7 @@ bool prune_unreachable(std::unique_ptr& ops) { op.cl = Op::_If; std::unique_ptr new_op = std::move(op.block0); op.block0 = std::move(op.block1); - op.block1 = std::make_unique(op.next->where, Op::_Nop); + op.block1 = std::make_unique(op.next->loc, Op::_Nop); new_op->last().next = std::move(ops); ops = std::move(new_op); } @@ -629,7 +629,7 @@ bool prune_unreachable(std::unique_ptr& ops) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{op.where, "unknown operation in prune_unreachable()"}; + throw ParseError(op.loc, "unknown operation in prune_unreachable()"); } if (reach) { return prune_unreachable(op.next); @@ -643,7 +643,7 @@ bool prune_unreachable(std::unique_ptr& ops) { void CodeBlob::prune_unreachable_code() { if (prune_unreachable(ops)) { - throw ParseError{loc, "control reaches end of function"}; + throw ParseError(fun_ref->loc, "control reaches end of function"); } } @@ -678,6 +678,16 @@ void Op::prepare_args(VarDescrList values) { } } +void Op::maybe_swap_builtin_args_to_compile() { + // in builtins.cpp, where optimizing constants are done, implementations assume that args are passed ltr (as declared); + // if a function has arg_order, called arguments might have been put on a stack not ltr, but in asm order; + // here we swap them back before calling FunctionBodyBuiltin compile, and also swap after + tolk_assert(arg_order_already_equals_asm()); + if (f_sym->method_name == "storeUint" || f_sym->method_name == "storeInt" || f_sym->method_name == "storeBool") { + std::swap(args[0], args[1]); + } +} + VarDescrList Op::fwd_analyze(VarDescrList values) { var_info.import_values(values); switch (cl) { @@ -704,10 +714,14 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { res.emplace_back(i); } AsmOpList tmp; - if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(tmp); // abstract interpretation of res := f (args) - } else { - std::get(f_sym->body)->compile(tmp, res, args, where); + if (!f_sym->is_asm_function()) { + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } + std::get(f_sym->body)->compile(tmp, res, args, loc); + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } int j = 0; for (var_idx_t i : left) { @@ -822,7 +836,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in fwd_analyze()"}; + throw ParseError(loc, "unknown operation in fwd_analyze()"); } if (next) { return next->fwd_analyze(std::move(values)); @@ -853,6 +867,10 @@ void Op::set_impure_flag() { flags |= _Impure; } +void Op::set_arg_order_already_equals_asm_flag() { + flags |= _ArgOrderAlreadyEqualsAsm; +} + bool Op::mark_noreturn() { switch (cl) { case _Nop: @@ -874,7 +892,32 @@ bool Op::mark_noreturn() { return set_noreturn(); case _Call: return set_noreturn(next->mark_noreturn() || does_function_always_throw(f_sym)); - case _If: + case _If: { + // this very-not-beautiful code does the following: + // replace `if (cond) { return; } else { block1; } next;` with `if (cond) { return; } block1; next` + // purpose: to make code like `if (...) { ... return; } else if (...) { ... return; } ...` act like without else + // similarly, `match (...) { v1 => { ... return; } ...}` is internally transformed to IF-ELSE + // (that's why such transformation is done at IR level, not in AST) + // without these code (without else removed), extra RETALT instructions are inserted, not necessary actually + // implementation is UGLY, because currently there is no way to perform IR replacements + // in the future, anyway, IR implementation should be rewritten, for easier traversing and replacement + // btw, now it doesn't work with `if (!...)` (v->is_ifnot), else "keyword" is not removed + if (block0->mark_noreturn() && !block1->is_empty()) { + VarDescrList block1_var_info = block1->var_info; // important to keep it + Op* last_in_block1 = block1.get(); + while (last_in_block1->next->cl != Op::_Nop) { // find the tail of a forward list of Ops + last_in_block1 = last_in_block1->next.get(); + } + last_in_block1->next = std::move(next); + next = std::move(block1); + block1 = std::make_unique(loc, Op::_Nop); + block1->var_info = std::move(block1_var_info); + } else { + block1->mark_noreturn(); + } + bool next_noreturn = next->mark_noreturn(); + return set_noreturn((block0->noreturn() && block1->noreturn()) || next_noreturn); + } case _TryCatch: // note, that & | (not && ||) here and below is mandatory to invoke both left and right calls return set_noreturn((static_cast(block0->mark_noreturn()) & static_cast(block1 && block1->mark_noreturn())) | static_cast(next->mark_noreturn())); @@ -891,7 +934,7 @@ bool Op::mark_noreturn() { return set_noreturn(next->mark_noreturn()); default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in mark_noreturn()"}; + throw ParseError(loc, "unknown operation in mark_noreturn()"); } } diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 2618ed26e..3dc39239b 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -52,30 +52,50 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, const std::string& push_op) { +// mirror the above operator<< formatting, but calculate resulting strlen +// used to align comments in Fift output +int AsmOp::SReg::calc_out_strlen() const { + int i = idx; + if (i >= 0) { + if (i < 10) { + return 2; + } else if (i < 16) { + return 3; + } else { + return 6; + } + } else if (i >= -2) { + return 5; + } else { + return 6; + } +} + + +AsmOp AsmOp::Const(SrcLocation loc, int arg, const std::string& push_op) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(loc, os.str()); } -AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { +AsmOp AsmOp::make_stk2(SrcLocation loc, int a, int b, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << str; int c = std::max(a, b) + 1; - return AsmOp::Custom(os.str(), c, c + delta); + return AsmOp::Custom(loc, os.str(), c, c + delta); } -AsmOp AsmOp::make_stk3(int a, int b, int c, const char* str, int delta) { +AsmOp AsmOp::make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; int m = std::max(a, std::max(b, c)) + 1; - return AsmOp::Custom(os.str(), m, m + delta); + return AsmOp::Custom(loc, os.str(), m, m + delta); } -AsmOp AsmOp::BlkSwap(int a, int b) { +AsmOp AsmOp::BlkSwap(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1 && b == 1) { - return AsmOp::Xchg(0, 1); + return AsmOp::Xchg(loc, 0, 1); } else if (a == 1) { if (b == 2) { os << "ROT"; @@ -91,125 +111,127 @@ AsmOp AsmOp::BlkSwap(int a, int b) { } else { os << a << " " << b << " BLKSWAP"; } - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::BlkPush(int a, int b) { +AsmOp AsmOp::BlkPush(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1) { - return AsmOp::Push(b); + return AsmOp::Push(loc, b); } else if (a == 2 && b == 1) { os << "2DUP"; } else { os << a << " " << b << " BLKPUSH"; } - return AsmOp::Custom(os.str(), b + 1, a + b + 1); + return AsmOp::Custom(loc, os.str(), b + 1, a + b + 1); } -AsmOp AsmOp::BlkDrop(int a) { +AsmOp AsmOp::BlkDrop(SrcLocation loc, int a) { std::ostringstream os; if (a == 1) { - return AsmOp::Pop(); + return AsmOp::Pop(loc, 0); } else if (a == 2) { os << "2DROP"; } else { os << a << " BLKDROP"; } - return AsmOp::Custom(os.str(), a, 0); + return AsmOp::Custom(loc, os.str(), a, 0); } -AsmOp AsmOp::BlkDrop2(int a, int b) { +AsmOp AsmOp::BlkDrop2(SrcLocation loc, int a, int b) { if (!b) { - return BlkDrop(a); + return BlkDrop(loc, a); } std::ostringstream os; os << a << " " << b << " BLKDROP2"; - return AsmOp::Custom(os.str(), a + b, b); + return AsmOp::Custom(loc, os.str(), a + b, b); } -AsmOp AsmOp::BlkReverse(int a, int b) { +AsmOp AsmOp::BlkReverse(SrcLocation loc, int a, int b) { std::ostringstream os; os << a << " " << b << " REVERSE"; - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::Tuple(int a) { +AsmOp AsmOp::Tuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("SINGLE", 1, 1); + return AsmOp::Custom(loc, "SINGLE", 1, 1); case 2: - return AsmOp::Custom("PAIR", 2, 1); + return AsmOp::Custom(loc, "PAIR", 2, 1); case 3: - return AsmOp::Custom("TRIPLE", 3, 1); + return AsmOp::Custom(loc, "TRIPLE", 3, 1); } std::ostringstream os; os << a << " TUPLE"; - return AsmOp::Custom(os.str(), a, 1); + return AsmOp::Custom(loc, os.str(), a, 1); } -AsmOp AsmOp::UnTuple(int a) { +AsmOp AsmOp::UnTuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("UNSINGLE", 1, 1); + return AsmOp::Custom(loc, "UNSINGLE", 1, 1); case 2: - return AsmOp::Custom("UNPAIR", 1, 2); + return AsmOp::Custom(loc, "UNPAIR", 1, 2); case 3: - return AsmOp::Custom("UNTRIPLE", 1, 3); + return AsmOp::Custom(loc, "UNTRIPLE", 1, 3); } std::ostringstream os; os << a << " UNTUPLE"; - return AsmOp::Custom(os.str(), 1, a); + return AsmOp::Custom(loc, os.str(), 1, a); } -AsmOp AsmOp::IntConst(const td::RefInt256& x) { +AsmOp AsmOp::IntConst(SrcLocation loc, const td::RefInt256& x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const(loc, "PUSHNAN"); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(loc, k, "PUSHPOW2"); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(loc, k, "PUSHPOW2DEC"); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(loc, k, "PUSHNEGPOW2"); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(x) + " PUSHINTX"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINTX"); } - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } -AsmOp AsmOp::BoolConst(bool f) { - return AsmOp::Const(f ? "TRUE" : "FALSE"); +AsmOp AsmOp::BoolConst(SrcLocation loc, bool f) { + return AsmOp::Const(loc, f ? "TRUE" : "FALSE"); } -AsmOp AsmOp::Parse(const std::string& custom_op) { +AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { if (custom_op == "NOP") { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } else if (custom_op == "SWAP") { - return AsmOp::Xchg(1); + return AsmOp::Xchg(loc, 1); } else if (custom_op == "DROP") { - return AsmOp::Pop(0); + return AsmOp::Pop(loc, 0); } else if (custom_op == "NIP") { - return AsmOp::Pop(1); + return AsmOp::Pop(loc, 1); } else if (custom_op == "DUP") { - return AsmOp::Push(0); + return AsmOp::Push(loc, 0); } else if (custom_op == "OVER") { - return AsmOp::Push(1); + return AsmOp::Push(loc, 1); + } else if (custom_op.ends_with(" PUSHINT") && custom_op[0] >= '1' && custom_op[0] <= '9' && custom_op.find(' ') == custom_op.rfind(' ')) { + return AsmOp::IntConst(loc, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); } else { - return AsmOp::Custom(custom_op); + return AsmOp::Custom(loc, custom_op); } } -AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { - auto res = Parse(custom_op); +AsmOp AsmOp::Parse(SrcLocation loc, std::string custom_op, int args, int retv) { + auto res = Parse(loc, custom_op); if (res.is_custom()) { res.a = args; res.b = retv; @@ -217,48 +239,50 @@ AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { return res; } -void AsmOp::out(std::ostream& os) const { +int AsmOp::out(std::ostream& os) const { if (!op.empty()) { os << op; - return; + return static_cast(op.size()); // return strlen to align a comment at the right } switch (t) { - case a_none: - break; + case a_nop: + case a_comment: + return 0; case a_xchg: if (!a && !(b & -2)) { os << (b ? "SWAP" : "NOP"); - break; + return b ? 4 : 3; } os << SReg(a) << ' ' << SReg(b) << " XCHG"; - break; + return SReg(a).calc_out_strlen() + 1 + SReg(b).calc_out_strlen() + 5; case a_push: if (!(a & -2)) { os << (a ? "OVER" : "DUP"); - break; + return a ? 4 : 3; } os << SReg(a) << " PUSH"; - break; + return SReg(a).calc_out_strlen() + 5; case a_pop: if (!(a & -2)) { os << (a ? "NIP" : "DROP"); - break; + return a ? 3 : 4; } os << SReg(a) << " POP"; - break; + return SReg(a).calc_out_strlen() + 4; default: - throw Fatal{"unknown assembler operation"}; + throw Fatal("unknown assembler operation"); } } -void AsmOp::out_indent_nl(std::ostream& os, bool no_eol) const { - for (int i = 0; i < indent; i++) { - os << " "; +int AsmOp::out_indented(std::ostream& os, bool print_src_line_above) const { + static int last_line_no = -1; + if (loc.is_defined() && print_src_line_above) { + loc.show_line_to_fif_output(os, indent, &last_line_no); } - out(os); - if (!no_eol) { - os << std::endl; + for (int i = 0; i < indent * 2; i++) { + os << ' '; } + return out(os) + indent * 2; } std::string AsmOp::to_string() const { @@ -271,16 +295,7 @@ std::string AsmOp::to_string() const { } } -bool AsmOpList::append(const std::vector& ops) { - for (const auto& op : ops) { - if (!append(op)) { - return false; - } - } - return true; -} - -const_idx_t AsmOpList::register_const(Const new_const) { +const_idx_t AsmOpList::register_const(td::RefInt256 new_const) { if (new_const.is_null()) { return not_const; } @@ -294,7 +309,7 @@ const_idx_t AsmOpList::register_const(Const new_const) { return (const_idx_t)idx; } -Const AsmOpList::get_const(const_idx_t idx) { +td::RefInt256 AsmOpList::get_const(const_idx_t idx) { if ((unsigned)idx < constants_.size()) { return constants_[idx]; } else { @@ -316,25 +331,27 @@ void AsmOpList::show_var_ext(std::ostream& os, std::pair } void AsmOpList::out(std::ostream& os, int mode) const { - if (!(mode & 2)) { - for (const auto& op : list_) { - op.out_indent_nl(os); - } - } else { - std::size_t n = list_.size(); - for (std::size_t i = 0; i < n; i++) { - const auto& op = list_[i]; - if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { - op.out_indent_nl(os, true); - os << '\t'; - do { - i++; - } while (i + 1 < n && list_[i + 1].is_comment()); - list_[i].out(os); - os << std::endl; - } else { - op.out_indent_nl(os, false); + std::size_t n = list_.size(); + for (std::size_t i = 0; i < n; i++) { + const AsmOp& op = list_[i]; + if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { + int len = op.out_indented(os, mode & Stack::_LineComments); + while (len < 28) { // align stack comments at the right + os << ' '; + len++; } + os << '\t'; + do { + i++; + } while (i + 1 < n && list_[i + 1].is_comment()); + list_[i].out(os); + os << std::endl; + } else if (op.is_comment()) { + op.out(os); + os << std::endl; + } else { + op.out_indented(os, mode & Stack::_LineComments); + os << std::endl; } } } @@ -344,7 +361,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op) { return false; } switch (op.t) { - case AsmOp::a_none: + case AsmOp::a_nop: return true; case AsmOp::a_xchg: return trans.apply_xchg(op.a, op.b, true); diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h new file mode 100644 index 000000000..36a806a09 --- /dev/null +++ b/tolk/ast-aux-data.h @@ -0,0 +1,75 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "lazy-helpers.h" +#include "tolk.h" + +/* + * This file contains a schema of aux_data inside ast_artificial_aux_vertex + * (it's a compiler-inserted vertex that can't occur in source code). + */ + +namespace tolk { + +// AuxData_ForceFiftLocation is created when transforming AST to IR; +// it wraps constants to force codegen location point to usage, not to init_val AST nodes +struct AuxData_ForceFiftLocation final : ASTAuxData { + SrcLocation forced_loc; + + explicit AuxData_ForceFiftLocation(SrcLocation forced_loc) + : forced_loc(forced_loc) { + } +}; + +// AuxData_LazyObjectLoadFields is a special auto-inserted vertex to load fields of a lazy struct; +// example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` +struct AuxData_LazyObjectLoadFields final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + TypePtr union_variant; // not just `o` but `match(o) { V1 => here }` + StructFieldPtr field_ref; // not just `o` but `match(o.field) { V1 => here }` + LazyStructLoadInfo load_info; // instructions, which fields to load, which to skip, etc. + + AuxData_LazyObjectLoadFields(LocalVarPtr var_ref, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo load_info) + : var_ref(var_ref), union_variant(union_variant), field_ref(field_ref), load_info(std::move(load_info)) { + } +}; + +// AuxData_LazyMatchForUnion wraps `match(lazy_var)` or its field +struct AuxData_LazyMatchForUnion final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + StructFieldPtr field_ref; // not `match(o)`, but `match(o.field)` + + AuxData_LazyMatchForUnion(LocalVarPtr var_ref, StructFieldPtr field_ref) + : var_ref(var_ref), field_ref(field_ref) { + } +}; + +struct AuxData_OnInternalMessage_getField final : ASTAuxData { + FunctionPtr f_onInternalMessage; + const std::string_view field_name; + + AuxData_OnInternalMessage_getField(FunctionPtr f_onInternalMessage, std::string_view field_name) + : f_onInternalMessage(f_onInternalMessage) + , field_name(field_name) { + } + + std::vector generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const; +}; + +} // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index fcaa11577..39cef55b7 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "ast-from-tokens.h" #include "ast.h" #include "type-system.h" #include "platform-utils.h" @@ -70,6 +69,16 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi "Use parenthesis to emphasize operator precedence."); } +// fire an error "Tolk does not have ++i operator" +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_no_increment_operator(SrcLocation loc, bool is_increment) { + std::string op_name = is_increment ? "increment" : "decrement"; + std::string op_wrong = is_increment ? "++" : "--"; + std::string op_right = is_increment ? "+=" : "-="; + throw ParseError(loc, std::string("Tolk has no ") + op_name + " operator\n" + + "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); +} + // diagnose when bitwise operators are used in a probably wrong way due to tricky precedence // example: "flags & 0xFF != 0" is equivalent to "flags & 1", most likely it's unexpected // the only way to suppress this error for the programmer is to use parenthesis @@ -77,12 +86,12 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi // that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") - if (rhs->type == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); } // handle "0 != flags & 0xFF" (lhs = "0 != flags") - if (lhs->type == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { + if (lhs->kind == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); } } @@ -106,34 +115,198 @@ static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType // diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { - if (rhs->type == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); } } -// replace (a == null) and similar to ast_is_null_check(a) (special AST vertex) +// replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { - bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; + bool has_null = v->get_lhs()->kind == ast_null_keyword || v->get_rhs()->kind == ast_null_keyword; bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); if (!replace) { return v; } - AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - return createV(v->loc, v_nullable, v->tok == tok_neq); + AnyExprV v_nullable = v->get_lhs()->kind == ast_null_keyword ? v->get_rhs() : v->get_lhs(); + AnyTypeV rhs_null_type = createV(v->loc, "null"); + return createV(v->loc, v_nullable, rhs_null_type, v->tok == tok_neq); } +// parse `123` / `0xFF` / `0b10001` to td::RefInt256 +static td::RefInt256 parse_tok_int_const(std::string_view text) { + bool bin = text[0] == '0' && text[1] == 'b'; + if (!bin) { + // this function parses decimal and hex numbers + return td::string_to_int256(static_cast(text)); + } + // parse a binary number; to make it simpler, don't allow too long numbers, it's impractical + if (text.size() > 64 + 2) { + return {}; + } + uint64_t result = 0; + for (char c : text.substr(2)) { // skip "0b" + result = (result << 1) | static_cast(c - '0'); + } + return td::make_refint(result); +} -/* - * - * PARSE SOURCE - * - */ + + +// -------------------------------------------- +// parsing type from tokens +// +// here we implement parsing types (mostly after colon) to AnyTypeV +// example: `var v: int` is leaf "int" +// example: `var v: (User?, [cell])` is tensor(nullable(leaf "User"), brackets(leaf "cell")) +// +// later, after all symbols are registered, types are resolved to TypePtr, see pipe-resolve-types.cpp +// + +static AnyTypeV parse_type_expression(Lexer& lex); + +static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { + lex.expect(tok_op, s_op); + std::vector sub_types; + while (true) { + if (lex.tok() == tok_cl) { // empty lists allowed + lex.next(); + break; + } + + sub_types.emplace_back(parse_type_expression(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_cl) { + // overcome the `>>` problem, like `Wrapper>`: + // treat token `>>` like two `>`; consume one here doing nothing (break) and leave the second `>` in a lexer + if (tok_cl == tok_gt && lex.tok() == tok_rshift) { + lex.hack_replace_rshift_with_one_triangle(); + break; + } + lex.unexpected(s_cl); + } + } + return sub_types; +} + +static std::vector parse_nested_type_list_in_triangles(Lexer& lex) { + return parse_nested_type_list(lex, tok_lt, "`<`", tok_gt, "`>` or `,`"); +} + +static AnyTypeV parse_simple_type(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + switch (lex.tok()) { + case tok_self: + case tok_identifier: + case tok_null: { + std::string_view text = lex.cur_str(); + lex.next(); + return createV(loc, text); + } + case tok_oppar: { + return createV(loc, parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`")); + } + case tok_opbracket: { + return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); + } + default: + lex.unexpected(""); + } +} + +static AnyTypeV parse_type_nullable(Lexer& lex) { + AnyTypeV result = parse_simple_type(lex); + + if (lex.tok() == tok_lt) { + std::vector args = parse_nested_type_list_in_triangles(lex); + std::vector outer_and_args; + outer_and_args.reserve(1 + args.size()); + outer_and_args.push_back(result); + outer_and_args.insert(outer_and_args.end(), args.begin(), args.end()); + result = createV(result->loc, std::move(outer_and_args)); + } + + if (lex.tok() == tok_question) { + lex.next(); + result = createV(result->loc, result); + } + + return result; +} + +static AnyTypeV parse_type_expression(Lexer& lex) { + if (lex.tok() == tok_bitwise_or) { // allow leading `|`, like in TypeScript + lex.next(); + } + AnyTypeV result = parse_type_nullable(lex); + + if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` + std::vector items; + items.emplace_back(result); + while (lex.tok() == tok_bitwise_or) { + lex.next(); + items.emplace_back(parse_type_nullable(lex)); + } + result = createV(result->loc, std::move(items)); + } + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` + lex.next(); + std::vector params_and_return; + if (auto p_tensor = result->try_as()) { + params_and_return.reserve(p_tensor->get_items().size()); + params_and_return.insert(params_and_return.begin(), p_tensor->get_items().begin(), p_tensor->get_items().end()); + } else { + params_and_return.reserve(2); + params_and_return.push_back(result); + } + params_and_return.push_back(parse_type_expression(lex)); + result = createV(result->loc, std::move(params_and_return)); + } + + return result; +} + +static AnyTypeV parse_type_from_tokens(Lexer& lex) { + return parse_type_expression(lex); +} + + + +// -------------------------------------------- +// parsing expressions and statements +// AnyExprV parse_expr(Lexer& lex); +AnyV parse_statement(Lexer& lex); + +static V parse_genericsT_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector genericsT_items; + lex.expect(tok_lt, "`<`"); + while (true) { + lex.check(tok_identifier, "T"); + SrcLocation locT = lex.cur_location(); + std::string_view nameT = lex.cur_str(); + lex.next(); + AnyTypeV default_type = nullptr; + if (lex.tok() == tok_assign) { // + lex.next(); + default_type = parse_type_expression(lex); + } + genericsT_items.emplace_back(createV(locT, nameT, default_type)); + if (lex.tok() != tok_comma) { + break; + } + lex.next(); + } + lex.expect(tok_gt, "`>`"); + return createV{loc, std::move(genericsT_items)}; +} -static AnyV parse_parameter(Lexer& lex, bool is_first) { +static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { SrcLocation loc = lex.cur_location(); // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) @@ -145,56 +318,77 @@ static AnyV parse_parameter(Lexer& lex, bool is_first) { // parameter name (or underscore for an unnamed parameter) std::string_view param_name; + bool is_self = false; if (lex.tok() == tok_identifier) { param_name = lex.cur_str(); } else if (lex.tok() == tok_self) { - if (!is_first) { - lex.error("`self` can only be the first parameter"); + if (!self_type) { + lex.error("`self` can only be the first parameter of a method"); } + is_self = true; param_name = "self"; } else if (lex.tok() != tok_underscore) { lex.unexpected("parameter name"); } lex.next(); - // parameter type after colon are mandatory - lex.expect(tok_colon, "`: `"); - TypePtr param_type = parse_type_from_tokens(lex); + // parameter type after colon is mandatory + AnyTypeV param_type = self_type; + if (!is_self) { + lex.expect(tok_colon, "`: `"); + param_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_colon) { + lex.error("`self` parameter should not have a type"); + } + + // optional default value + AnyExprV default_value = nullptr; + if (lex.tok() == tok_assign && !is_self) { // `a: int = 0` + if (declared_as_mutate) { + lex.error("`mutate` parameter can't have a default value"); + } + lex.next(); + default_value = parse_expr(lex); + } - return createV(loc, param_name, param_type, declared_as_mutate); + return createV(loc, param_name, param_type, default_value, declared_as_mutate); } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_global, "`global`"); lex.check(tok_identifier, "global variable name"); auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_colon, "`:`"); - TypePtr declared_type = parse_type_from_tokens(lex); + AnyTypeV declared_type = parse_type_from_tokens(lex); if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split globals on separate lines"); } if (lex.tok() == tok_assign) { lex.error("assigning to a global is not allowed at declaration"); } - lex.expect(tok_semicolon, "`;`"); + + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to global"); + } + } + return createV(loc, v_ident, declared_type); } static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_const, "`const`"); lex.check(tok_identifier, "constant name"); auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; if (lex.tok() == tok_colon) { lex.next(); declared_type = parse_type_from_tokens(lex); @@ -204,20 +398,149 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to constant"); + } + } + return createV(loc, v_ident, declared_type, init_value); } +static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_type, "`type`"); + lex.check(tok_identifier, "type name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'type Response' + genericsT_list = parse_genericsT_list(lex); + } + + lex.expect(tok_assign, "`=`"); + AnyTypeV underlying_type = parse_type_from_tokens(lex); + + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to type alias"); + } + } + + return createV(loc, v_ident, genericsT_list, underlying_type); +} + +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool allow_lateinit) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable, false); + if (lex.tok() == tok_clpar) { + lex.next(); + return first; + } + std::vector args(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_opbracket) { + lex.next(); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); + while (lex.tok() == tok_comma) { + lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_identifier) { + auto v_ident = createV(loc, lex.cur_str()); + AnyTypeV declared_type = nullptr; + bool marked_as_redef = false; + bool is_lateinit = false; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_redef) { + lex.next(); + marked_as_redef = true; + } + if (lex.tok() == tok_semicolon && allow_lateinit) { + if (declared_type == nullptr) { + lex.error("provide a type for a variable, because its default value is omitted:\n> var " + static_cast(v_ident->name) + ": ;"); + } + is_lateinit = true; + } + return createV(loc, v_ident, declared_type, is_immutable, is_lateinit, marked_as_redef); + } + if (lex.tok() == tok_underscore) { + AnyTypeV declared_type = nullptr; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + return createV(loc, createV(loc, ""), declared_type, true, false, false); + } + lex.unexpected("variable name"); +} + +static AnyExprV parse_local_vars_declaration(Lexer& lex, bool allow_lateinit) { + SrcLocation loc = lex.cur_location(); + bool is_immutable = lex.tok() == tok_val; + lex.next(); + + AnyExprV lhs = parse_var_declaration_lhs(lex, is_immutable, allow_lateinit); + if (lex.tok() != tok_assign) { + if (auto lhs_var = lhs->try_as(); lhs_var && lhs_var->is_lateinit) { + return lhs; // just ast_local_var_lhs inside AST tree + } + lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); + } + lex.next(); + AnyExprV rhs = parse_expr(lex); + + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split variables on separate lines"); + } + return createV(loc, createV(loc, lhs), rhs); +} + // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` -static V parse_parameter_list(Lexer& lex) { +// for methods like `fun builder.storeUint(self, i: int)`, receiver_type = builder (type of self) +static V parse_parameter_list(Lexer& lex, AnyTypeV receiver_type) { SrcLocation loc = lex.cur_location(); std::vector params; lex.expect(tok_oppar, "parameter list"); if (lex.tok() != tok_clpar) { - params.push_back(parse_parameter(lex, true)); + params.push_back(parse_parameter(lex, receiver_type)); while (lex.tok() == tok_comma) { lex.next(); - params.push_back(parse_parameter(lex, false)); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } + params.push_back(parse_parameter(lex, nullptr)); } } lex.expect(tok_clpar, "`)`"); @@ -247,6 +570,9 @@ static V parse_argument_list(Lexer& lex) { args.push_back(parse_argument(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_argument(lex)); } } @@ -274,6 +600,165 @@ static V parse_maybe_instantiationTs_after_identifier(L } } +static V parse_object_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + if (lex.tok() == tok_comma || lex.tok() == tok_clbrace) { + auto v_same_ident = createV(v_ident->loc, v_ident->name); + auto v_same_expr = createV(v_ident->loc, v_same_ident, nullptr); + return createV(loc, v_ident, v_same_expr); + } + + lex.expect(tok_colon, "`:`"); + return createV(loc, v_ident, parse_expr(lex)); +} + +static V parse_object_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector fields; + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_object_field(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(fields)); +} + +// `throw code` / `throw (code)` / `throw (code, arg)` +// it's technically a statement (can't occur "in any place of expression"), +// but inside `match` arm it can appear without semicolon: `pattern => throw 123` +static AnyV parse_throw_expression(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_throw, "`throw`"); + + AnyExprV thrown_code, thrown_arg; + if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) + lex.next(); + thrown_code = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.next(); + thrown_arg = parse_expr(lex); + } else { + thrown_arg = createV(loc); + } + lex.expect(tok_clpar, "`)`"); + } else { // throw code + thrown_code = parse_expr(lex); + thrown_arg = createV(loc); + } + + return createV(loc, thrown_code, thrown_arg); +} + +// `pattern => body` inside `match` +static V parse_match_arm(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + MatchArmKind pattern_kind = static_cast(-1); + AnyTypeV exact_type = nullptr; + AnyExprV pattern_expr = nullptr; + + Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); + try { + exact_type = parse_type_from_tokens(lex); + pattern_kind = MatchArmKind::exact_type; + } catch (const ParseError&) { + } + if (!exact_type || lex.tok() != tok_double_arrow) { + exact_type = nullptr; + lex.restore_position(backup); + try { + pattern_expr = parse_expr(lex); + pattern_kind = MatchArmKind::const_expression; // any expr at parsing, should result in const int/bool + } catch (const ParseError&) { + } + } + if (!exact_type && !pattern_expr && lex.tok() == tok_else) { + lex.next(); + pattern_kind = MatchArmKind::else_branch; + } + + if (pattern_kind == static_cast(-1)) { + lex.restore_position(backup); + throw ParseError(loc, "expected or in `match` before `=>`"); + } + lex.expect(tok_double_arrow, "`=>`"); + + V body = nullptr; + if (lex.tok() == tok_opbrace) { // pattern => { ... } + AnyV v_block = parse_statement(lex); + body = createV(v_block->loc, v_block); + } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) + AnyV v_throw = parse_throw_expression(lex); + AnyV v_block = createV(v_throw->loc, v_throw->loc, {v_throw}); + body = createV(v_block->loc, v_block); + } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) + lex.next(); + AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); + AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); + body = createV(v_block->loc, v_block); + } else { + AnyExprV unbraced_expr = parse_expr(lex); + AnyV v_block = createV(unbraced_expr->loc, unbraced_expr->loc, {createV(unbraced_expr->loc, unbraced_expr)}); + body = createV(unbraced_expr->loc, v_block); + } + + if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr + pattern_expr = createV(loc); + } + return createV(loc, pattern_kind, exact_type, pattern_expr, body); +} + +static V parse_match_expression(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_match, "`match`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` + ? parse_local_vars_declaration(lex, false) + : parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + + std::vector subject_and_arms = {subject}; + lex.expect(tok_opbrace, "`{`"); + while (lex.tok() != tok_clbrace) { + auto v_arm = parse_match_arm(lex); + subject_and_arms.push_back(v_arm); + + // after `pattern => { ... }` comma is optional, after `pattern => expr` mandatory + bool was_comma = lex.tok() == tok_comma; // trailing comma is allowed always + bool was_unbraced = v_arm->get_body()->get_block_statement()->size() == 1 && v_arm->get_body()->get_block_statement()->get_item(0)->kind == ast_braced_yield_result; + if (was_comma) { + lex.next(); + } + if (lex.tok() == tok_clbrace) { + break; + } + if (!was_comma && was_unbraced) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + return createV(loc, std::move(subject_and_arms)); +} + +static V parse_lazy_operator(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_lazy, "`lazy`"); + + AnyExprV expr = parse_expr(lex); + return createV(loc, expr); +} + // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -292,28 +777,37 @@ static AnyExprV parse_expr100(Lexer& lex) { std::vector items(1, first); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); + if (items.size() == 1) { // we can reach here for 1 element with trailing comma: `(item, )` + return items[0]; // then just return item, not a 1-element tensor, + } // since 1-element tensors won't be type compatible with item's type return createV(loc, std::move(items)); } case tok_opbracket: { lex.next(); if (lex.tok() == tok_clbracket) { lex.next(); - return createV(loc, {}); + return createV(loc, {}); } std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(items)); + return createV(loc, std::move(items)); } case tok_int_const: { std::string_view orig_str = lex.cur_str(); - td::RefInt256 intval = td::string_to_int256(static_cast(orig_str)); + td::RefInt256 intval = parse_tok_int_const(orig_str); if (intval.is_null() || !intval->signed_fits_bits(257)) { lex.error("invalid integer constant"); } @@ -323,12 +817,7 @@ static AnyExprV parse_expr100(Lexer& lex) { case tok_string_const: { std::string_view str_val = lex.cur_str(); lex.next(); - char modifier = 0; - if (lex.tok() == tok_string_modifier) { - modifier = lex.cur_str()[0]; - lex.next(); - } - return createV(loc, str_val, modifier); + return createV(loc, str_val); } case tok_underscore: { lex.next(); @@ -358,14 +847,33 @@ static AnyExprV parse_expr100(Lexer& lex) { if (lex.tok() == tok_lt) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); } + if (lex.tok() == tok_opbrace) { + AnyTypeV type_node = createV(v_ident->loc, v_ident->name); // `Pair { ... }` + if (v_instantiationTs) { // `Pair { ... }` + std::vector ident_and_args; + ident_and_args.reserve(1 + v_instantiationTs->size()); + ident_and_args.push_back(type_node); + for (int i = 0; i < v_instantiationTs->size(); ++i) { + ident_and_args.push_back(v_instantiationTs->get_item(i)->type_node); + } + type_node = createV(v_ident->loc, std::move(ident_and_args)); + } + return createV(loc, type_node, parse_object_body(lex)); + } return createV(loc, v_ident, v_instantiationTs); } + case tok_opbrace: + return createV(loc, nullptr, parse_object_body(lex)); + case tok_match: + return parse_match_expression(lex); + case tok_lazy: + return parse_lazy_operator(lex); default: lex.unexpected(""); } } -// parse E(...) and E! having parsed E already (left-to-right) +// parse E(...) / E! / E++ / E-- having parsed E already (left-to-right) static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { while (true) { if (lex.tok() == tok_oppar) { @@ -373,6 +881,8 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { } else if (lex.tok() == tok_logical_not) { lex.next(); lhs = createV(lhs->loc, lhs); + } else if (lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { + fire_error_no_increment_operator(lex.cur_location(), lex.tok() == tok_double_plus); } else { break; } @@ -383,7 +893,7 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { // parse E(...) and E! (left-to-right) static AnyExprV parse_expr90(Lexer& lex) { AnyExprV res = parse_expr100(lex); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { res = parse_fun_call_postfix(lex, res); } return res; @@ -410,7 +920,7 @@ static AnyExprV parse_expr80(Lexer& lex) { lex.unexpected("method name"); } lhs = createV(loc, lhs, v_ident, v_instantiationTs); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { lhs = parse_fun_call_postfix(lex, lhs); } } @@ -427,17 +937,32 @@ static AnyExprV parse_expr75(Lexer& lex) { AnyExprV rhs = parse_expr75(lex); return createV(loc, operator_name, t, rhs); } + if (t == tok_double_minus || t == tok_double_plus) { + SrcLocation loc = lex.cur_location(); + lex.next(); + parse_expr75(lex); + fire_error_no_increment_operator(loc, t == tok_double_plus); + } return parse_expr80(lex); } -// parse E as +// parse E as / is / !is static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); if (lex.tok() == tok_as) { SrcLocation loc = lex.cur_location(); lex.next(); - TypePtr cast_to_type = parse_type_from_tokens(lex); + AnyTypeV cast_to_type = parse_type_from_tokens(lex); lhs = createV(loc, lhs, cast_to_type); + } else if (lex.tok() == tok_is) { + SrcLocation loc = lex.cur_location(); + lex.next(); + AnyTypeV rhs_type = parse_type_from_tokens(lex); + bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + if (is_negated) { + lhs = lhs->as()->get_expr(); + } + lhs = createV(loc, lhs, rhs_type, is_negated); } return lhs; } @@ -572,90 +1097,26 @@ AnyExprV parse_expr(Lexer& lex) { return parse_expr10(lex); } -AnyV parse_statement(Lexer& lex); - -static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { - SrcLocation loc = lex.cur_location(); - if (lex.tok() == tok_oppar) { - lex.next(); - AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); - if (lex.tok() == tok_clpar) { - lex.next(); - return first; - } - std::vector args(1, first); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_opbracket) { - lex.next(); - std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_identifier) { - auto v_ident = createV(loc, lex.cur_str()); - TypePtr declared_type = nullptr; - bool marked_as_redef = false; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } else if (lex.tok() == tok_redef) { - lex.next(); - marked_as_redef = true; - } - return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); - } - if (lex.tok() == tok_underscore) { - TypePtr declared_type = nullptr; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } - return createV(loc, createV(loc, ""), declared_type, true, false); - } - lex.unexpected("variable name"); -} - -static AnyV parse_local_vars_declaration_assignment(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - bool is_immutable = lex.tok() == tok_val; - lex.next(); - - AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); - if (lex.tok() != tok_assign) { - lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); - } - lex.next(); - AnyExprV rhs = parse_expr(lex); - - if (lex.tok() == tok_comma) { - lex.error("multiple declarations are not allowed, split variables on separate lines"); - } - lex.expect(tok_semicolon, "`;`"); - return createV(loc, lhs, rhs); -} - -static V parse_sequence(Lexer& lex) { +static V parse_block_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_opbrace, "`{`"); std::vector items; while (lex.tok() != tok_clbrace) { - items.push_back(parse_statement(lex)); + AnyV v = parse_statement(lex); + items.push_back(v); + if (lex.tok() == tok_clbrace) { + break; + } + bool does_end_with_brace = v->kind == ast_if_statement || v->kind == ast_while_statement + || v->kind == ast_match_expression + || v->kind == ast_try_catch_statement || v->kind == ast_repeat_statement || v->kind == ast_block_statement; + if (!does_end_with_brace) { + lex.expect(tok_semicolon, "`;`"); + } } SrcLocation loc_end = lex.cur_location(); lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, items); + return createV(loc, loc_end, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { @@ -664,7 +1125,6 @@ static AnyV parse_return_statement(Lexer& lex) { AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) ? createV(lex.cur_location()) : parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); return createV(loc, child); } @@ -676,18 +1136,18 @@ static AnyV parse_if_statement(Lexer& lex) { AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V if_body = parse_sequence(lex); - V else_body = nullptr; + V if_body = parse_block_statement(lex); + V else_body = nullptr; if (lex.tok() == tok_else) { // else if(e) { } or else { } lex.next(); if (lex.tok() == tok_if) { AnyV v_inner_if = parse_if_statement(lex); - else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); } else { - else_body = parse_sequence(lex); + else_body = parse_block_statement(lex); } } else { // no 'else', create empty block - else_body = createV(lex.cur_location(), lex.cur_location(), {}); + else_body = createV(lex.cur_location(), lex.cur_location(), {}); } return createV(loc, false, cond, if_body, else_body); } @@ -698,7 +1158,7 @@ static AnyV parse_repeat_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } @@ -708,19 +1168,18 @@ static AnyV parse_while_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } static AnyV parse_do_while_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_do, "`do`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); lex.expect(tok_while, "`while`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - lex.expect(tok_semicolon, "`;`"); return createV(loc, body, cond); } @@ -745,30 +1204,6 @@ static AnyExprV create_catch_underscore_variable(const Lexer& lex) { return createV(lex.cur_location(), v_ident, nullptr); } -static AnyV parse_throw_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.expect(tok_throw, "`throw`"); - - AnyExprV thrown_code, thrown_arg; - if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) - lex.next(); - thrown_code = parse_expr(lex); - if (lex.tok() == tok_comma) { - lex.next(); - thrown_arg = parse_expr(lex); - } else { - thrown_arg = createV(loc); - } - lex.expect(tok_clpar, "`)`"); - } else { // throw code - thrown_code = parse_expr(lex); - thrown_arg = createV(loc); - } - - lex.expect(tok_semicolon, "`;`"); - return createV(loc, thrown_code, thrown_arg); -} - static AnyV parse_assert_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_assert, "`assert`"); @@ -786,14 +1221,13 @@ static AnyV parse_assert_statement(Lexer& lex) { thrown_code = parse_expr(lex); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, cond, thrown_code); } static AnyV parse_try_catch_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_try, "`try`"); - V try_body = parse_sequence(lex); + V try_body = parse_block_statement(lex); std::vector catch_args; lex.expect(tok_catch, "`catch`"); @@ -814,7 +1248,7 @@ static AnyV parse_try_catch_statement(Lexer& lex) { } V catch_expr = createV(catch_loc, std::move(catch_args)); - V catch_body = parse_sequence(lex); + V catch_body = parse_block_statement(lex); return createV(loc, try_body, catch_expr, catch_body); } @@ -822,9 +1256,9 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex); + return parse_local_vars_declaration(lex, true); case tok_opbrace: - return parse_sequence(lex); + return parse_block_statement(lex); case tok_return: return parse_return_statement(lex); case tok_if: @@ -836,29 +1270,29 @@ AnyV parse_statement(Lexer& lex) { case tok_while: return parse_while_statement(lex); case tok_throw: - return parse_throw_statement(lex); + return parse_throw_expression(lex); case tok_assert: return parse_assert_statement(lex); case tok_try: return parse_try_catch_statement(lex); - case tok_semicolon: { - SrcLocation loc = lex.cur_location(); - lex.next(); - return createV(loc); - } + case tok_semicolon: + return createV(lex.cur_location()); case tok_break: case tok_continue: lex.error("break/continue from loops are not supported yet"); - default: { - AnyExprV expr = parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); - return expr; - } + default: + return parse_expr(lex); } } + +// -------------------------------------------- +// parsing top-level declarations +// + + static AnyV parse_func_body(Lexer& lex) { - return parse_sequence(lex); + return parse_block_statement(lex); } static AnyV parse_asm_func_body(Lexer& lex, V param_list) { @@ -866,7 +1300,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.expect(tok_asm, "`asm`"); size_t n_params = param_list->size(); if (n_params > 16) { - throw ParseError{loc, "assembler built-in function can have at most 16 arguments"}; + throw ParseError(loc, "assembler built-in function can have at most 16 arguments"); } std::vector arg_order, ret_order; if (lex.tok() == tok_oppar) { @@ -893,31 +1327,12 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.check(tok_string_const, "\"ASM COMMAND\""); while (lex.tok() == tok_string_const) { std::string_view asm_command = lex.cur_str(); - asm_commands.push_back(createV(lex.cur_location(), asm_command, 0)); + asm_commands.push_back(createV(lex.cur_location(), asm_command)); lex.next(); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } -static AnyV parse_genericsT_list(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - std::vector genericsT_items; - lex.expect(tok_lt, "`<`"); - while (true) { - lex.check(tok_identifier, "T"); - std::string_view nameT = lex.cur_str(); - genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); - lex.next(); - if (lex.tok() != tok_comma) { - break; - } - lex.next(); - } - lex.expect(tok_gt, "`>`"); - return createV{loc, std::move(genericsT_items)}; -} - static V parse_annotation(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.check(tok_annotation_at, "`@`"); @@ -933,6 +1348,9 @@ static V parse_annotation(Lexer& lex) { args.push_back(parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); @@ -944,40 +1362,58 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "unknown annotation " + static_cast(name)); case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: + case AnnotationKind::noinline: case AnnotationKind::pure: - case AnnotationKind::deprecated: if (v_arg) { throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); } v_arg = createV(loc, {}); break; + case AnnotationKind::deprecated: + case AnnotationKind::custom: + // allowed with and without arguments; it's IDE-only, the compiler doesn't analyze @deprecated + break; case AnnotationKind::method_id: - if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->type != ast_int_const) { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; + case AnnotationKind::overflow1023_policy: + case AnnotationKind::on_bounced_policy: { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { + throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); + } + break; + } } - return createV(loc, kind, v_arg); + return createV(loc, name, kind, v_arg); } -static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { +static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations, bool is_contract_getter) { SrcLocation loc = lex.cur_location(); - bool is_get_method = lex.tok() == tok_get; - lex.next(); - if (is_get_method && lex.tok() == tok_fun) { - lex.next(); // 'get f()' and 'get fun f()' both correct + lex.expect(tok_fun, "`fun`"); + + AnyTypeV receiver_type = nullptr; + auto backup = lex.save_parsing_position(); + try { + receiver_type = parse_type_expression(lex); + lex.expect(tok_dot, ""); + } catch (const ParseError&) { + receiver_type = nullptr; + lex.restore_position(backup); } lex.check(tok_identifier, "function name identifier"); std::string_view f_name = lex.cur_str(); - bool is_entrypoint = + bool is_entrypoint = !receiver_type && ( f_name == "main" || f_name == "onInternalMessage" || f_name == "onExternalMessage" || - f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"; - bool is_FunC_entrypoint = + f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall" || + f_name == "onBouncedMessage"); + bool is_FunC_entrypoint = !receiver_type && ( f_name == "recv_internal" || f_name == "recv_external" || - f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"; + f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"); if (is_FunC_entrypoint) { lex.error("this is a reserved FunC/Fift identifier; you need `onInternalMessage`"); } @@ -987,14 +1423,14 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector genericsT_list = nullptr; if (lex.tok() == tok_lt) { // 'fun f' - genericsT_list = parse_genericsT_list(lex)->as(); + genericsT_list = parse_genericsT_list(lex); } - V v_param_list = parse_parameter_list(lex)->as(); + V v_param_list = parse_parameter_list(lex, receiver_type)->as(); bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); - TypePtr ret_type = nullptr; + AnyTypeV ret_type = nullptr; bool returns_self = false; if (lex.tok() == tok_colon) { // : (if absent, it means "auto infer", not void) lex.next(); @@ -1002,19 +1438,19 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), "void"); + lex.next(); } else { ret_type = parse_type_from_tokens(lex); } } - if (is_entrypoint && (is_get_method || genericsT_list || n_mutate_params || accepts_self)) { + if (is_entrypoint && (is_contract_getter || genericsT_list || n_mutate_params)) { throw ParseError(loc, "invalid declaration of a reserved function"); } - if (is_get_method && (genericsT_list || n_mutate_params || accepts_self)) { - throw ParseError(loc, "get methods can't have `mutate` and `self` params"); + if (is_contract_getter && (genericsT_list || n_mutate_params || receiver_type)) { + throw ParseError(loc, "invalid declaration of a get method"); } AnyV v_body = nullptr; @@ -1022,7 +1458,6 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location()); lex.next(); - lex.expect(tok_semicolon, "`;`"); } else if (lex.tok() == tok_opbrace) { v_body = parse_func_body(lex); } else if (lex.tok() == tok_asm) { @@ -1038,8 +1473,8 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorkind) { case AnnotationKind::inline_simple: - flags |= FunctionData::flagInline; + inline_mode = FunctionInlineMode::inlineViaFif; // maybe will be replaced by inlineInPlace later break; case AnnotationKind::inline_ref: - flags |= FunctionData::flagInlineRef; + inline_mode = FunctionInlineMode::inlineRef; + break; + case AnnotationKind::noinline: + inline_mode = FunctionInlineMode::noInline; break; case AnnotationKind::pure: flags |= FunctionData::flagMarkedAsPure; break; case AnnotationKind::method_id: { - if (is_get_method || genericsT_list || is_entrypoint || n_mutate_params || accepts_self) { + if (is_contract_getter || genericsT_list || receiver_type || is_entrypoint || n_mutate_params || accepts_self) { v_annotation->error("@method_id can be specified only for regular functions"); } auto v_int = v_annotation->get_arg()->get_item(0)->as(); if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { v_int->error("invalid integer constant"); } - method_id = v_int->intval; + tvm_method_id = v_int->intval; + break; + } + case AnnotationKind::on_bounced_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "manual") { + flags |= FunctionData::flagManualOnBounce; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + if (f_name != "onInternalMessage") { + v_annotation->error("this annotation is applicable only to onInternalMessage()"); + } break; } case AnnotationKind::deprecated: - // no special handling + case AnnotationKind::custom: break; default: @@ -1080,7 +1531,93 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); + return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags, inline_mode); +} + +static AnyV parse_struct_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_colon, "`: `"); + AnyTypeV declared_type = parse_type_from_tokens(lex); + + AnyExprV default_value = nullptr; + if (lex.tok() == tok_assign) { // `id: int = 3` + lex.next(); + default_value = parse_expr(lex); + } + + return createV(loc, v_ident, default_value, declared_type); +} + +static V parse_struct_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector fields; + + if (lex.tok() == tok_opbrace) { // `struct A` equal to `struct A {}` + lex.next(); + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_struct_field(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } + } + lex.expect(tok_clbrace, "`}`"); + } + + return createV(loc, std::move(fields)); +} + +static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_struct, "`struct`"); + + AnyExprV opcode = nullptr; + if (lex.tok() == tok_oppar) { // struct(0x0012) CounterIncrement + lex.next(); + lex.check(tok_int_const, "opcode `0x...` or `0b...`"); + std::string_view opcode_str = lex.cur_str(); + if (!opcode_str.starts_with("0x") && !opcode_str.starts_with("0b")) { + lex.unexpected("opcode `0x...` or `0b...`"); + } + opcode = createV(lex.cur_location(), parse_tok_int_const(opcode_str), opcode_str); + lex.next(); + lex.expect(tok_clpar, "`)`"); + } else { + opcode = createV(lex.cur_location()); + } + + lex.check(tok_identifier, "identifier"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'struct Wrapper' + genericsT_list = parse_genericsT_list(lex); + } + + StructData::Overflow1023Policy overflow1023_policy = StructData::Overflow1023Policy::not_specified; + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + case AnnotationKind::overflow1023_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "suppress") { + overflow1023_policy = StructData::Overflow1023Policy::suppress; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + break; + } + default: + v_annotation->error("this annotation is not applicable to struct"); + } + } + + return createV(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { @@ -1105,12 +1642,17 @@ static AnyV parse_import_directive(Lexer& lex) { if (rel_filename.empty()) { lex.error("imported file name is an empty string"); } - auto v_str = createV(lex.cur_location(), rel_filename, 0); + auto v_str = createV(lex.cur_location(), rel_filename); lex.next(); return createV(loc, v_str); // semicolon is not necessary } -// the main (exported) function + +// -------------------------------------------- +// parse .tolk source file to AST +// (the main, exported, function) +// + AnyV parse_src_file_to_ast(const SrcFile* file) { std::vector toplevel_declarations; std::vector> annotations; @@ -1148,21 +1690,35 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_constant_declaration(lex, annotations)); annotations.clear(); break; + case tok_type: + toplevel_declarations.push_back(parse_type_alias_declaration(lex, annotations)); + annotations.clear(); + break; case tok_fun: - case tok_get: - toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, false)); + annotations.clear(); + break; + case tok_struct: + toplevel_declarations.push_back(parse_struct_declaration(lex, annotations)); annotations.clear(); break; case tok_export: - case tok_struct: case tok_enum: case tok_operator: case tok_infix: lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + case tok_identifier: + if (lex.cur_str() == "get") { // top-level "get", contract getter + lex.next(); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, true)); + annotations.clear(); + break; + } + // fallthrough default: - lex.unexpected("fun or get"); + lex.unexpected("top-level declaration"); } } diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 5103cc923..581babee3 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -60,6 +60,12 @@ class ASTReplacer { return v_mutable; } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBlockOfStatements* v) { + auto* v_mutable = const_cast(v); + v_mutable->child_block_statement = replace(v_mutable->child_block_statement->as()); + return v_mutable; + } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementUnary* v) { auto* v_mutable = const_cast(v); v_mutable->child = replace(v_mutable->child); @@ -88,8 +94,11 @@ class ASTReplacerInFunctionBody : public ASTReplacer { // expressions virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -108,11 +117,17 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } - virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -123,11 +138,14 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyV replace(V v) { return replace_children(v); } AnyExprV replace(AnyExprV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); + case ast_braced_expression: return replace(v->as()); + case ast_braced_yield_result: return replace(v->as()); + case ast_artificial_aux_vertex: return replace(v->as()); case ast_tensor: return replace(v->as()); - case ast_typed_tuple: return replace(v->as()); + case ast_bracket_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); case ast_local_var_lhs: return replace(v->as()); case ast_local_vars_declaration: return replace(v->as()); @@ -146,17 +164,23 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_binary_operator: return replace(v->as()); case ast_ternary_operator: return replace(v->as()); case ast_cast_as_operator: return replace(v->as()); + case ast_is_type_operator: return replace(v->as()); case ast_not_null_operator: return replace(v->as()); - case ast_is_null_check: return replace(v->as()); + case ast_lazy_operator: return replace(v->as()); + case ast_match_expression: return replace(v->as()); + case ast_match_arm: return replace(v->as()); + case ast_object_field: return replace(v->as()); + case ast_object_body: return replace(v->as()); + case ast_object_literal: return replace(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacerInFunctionBody::replace"); } } AnyV replace(AnyV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_statement: return replace(v->as()); - case ast_sequence: return replace(v->as()); + case ast_block_statement: return replace(v->as()); case ast_return_statement: return replace(v->as()); case ast_if_statement: return replace(v->as()); case ast_repeat_statement: return replace(v->as()); @@ -167,7 +191,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_try_catch_statement: return replace(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTReplacer::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacer::replace"); #endif default: { // be very careful, don't forget to handle all statements (not expressions) above! @@ -180,13 +204,15 @@ class ASTReplacerInFunctionBody : public ASTReplacer { public: virtual bool should_visit_function(FunctionPtr fun_ref) = 0; - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + virtual void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { replace(v_function->get_body()); } }; const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 16bbbeb8d..19956e467 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -21,21 +21,8 @@ namespace tolk { -class ASTReplicator { -protected: - virtual AnyV clone(AnyV v) = 0; - virtual AnyExprV clone(AnyExprV v) = 0; - virtual TypePtr clone(TypePtr) = 0; - -public: - virtual ~ASTReplicator() = default; -}; - -class ASTReplicatorFunction : public ASTReplicator { -protected: - using parent = ASTReplicatorFunction; - - std::vector clone(const std::vector& items) { +class ASTReplicator final { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyV item : items) { @@ -44,7 +31,7 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } - std::vector clone(const std::vector& items) { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyExprV item : items) { @@ -53,141 +40,249 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } + static std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyTypeV item : items) { + result.push_back(clone(item)); + } + return result; + } + + // types + + static V clone(V v) { + return createV(v->loc, v->text); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_inner())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_params_and_return())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_variants())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_inner_and_args())); + } + // expressions - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { + return createV(v->loc, clone(v->get_block_statement())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); + } + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_items())); + static V clone(V v) { + return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->declared_type), v->is_immutable, v->marked_as_redef); + static V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->is_lateinit, v->marked_as_redef); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->intval, v->orig_str); } - virtual V clone(V v) { - return createV(v->loc, v->str_val, v->modifier); + static V clone(V v) { + return createV(v->loc, v->str_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->bool_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_arguments())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); + static V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), v->is_negated); + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_all_children())); + } + static V clone(V v) { + return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } + static V clone(V v) { + return createV(v->loc, clone(v->type_node), clone(v->get_body())); } // statements - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { - return createV(v->loc, v->loc_end, clone(v->get_items())); + static V clone(V v) { + return createV(v->loc, v->loc_end, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_return_value())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); } - // other + // other (common) - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->name); } - virtual V clone(V v) { - return createV(v->loc, clone(v->substituted_type)); + static V clone(V v) { + return createV(v->loc, v->nameT, clone(v->default_type_node)); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + static V clone(V v) { + return createV(v->loc, clone(v->type_node)); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { - return createV(v->loc, v->param_name, clone(v->declared_type), v->declared_as_mutate); + static V clone(V v) { + return createV(v->loc, v->param_name, clone(v->type_node), v->default_value ? clone(v->default_value) : nullptr, v->declared_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_params())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } + + + static AnyV clone(AnyV v) { + switch (v->kind) { + case ast_empty_statement: return clone(v->as()); + case ast_block_statement: return clone(v->as()); + case ast_return_statement: return clone(v->as()); + case ast_if_statement: return clone(v->as()); + case ast_repeat_statement: return clone(v->as()); + case ast_while_statement: return clone(v->as()); + case ast_do_while_statement: return clone(v->as()); + case ast_throw_statement: return clone(v->as()); + case ast_assert_statement: return clone(v->as()); + case ast_try_catch_statement: return clone(v->as()); + case ast_asm_body: return clone(v->as()); + // other AST nodes that can be children of ast nodes of function/struct body + case ast_identifier: return clone(v->as()); + case ast_genericsT_item: return clone(v->as()); + case ast_genericsT_list: return clone(v->as()); + case ast_instantiationT_item: return clone(v->as()); + case ast_instantiationT_list: return clone(v->as()); + case ast_parameter: return clone(v->as()); + case ast_parameter_list: return clone(v->as()); + case ast_struct_field: return clone(v->as()); + case ast_struct_body: return clone(v->as()); + + default: { + // be very careful, don't forget to handle all statements/other (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return clone(as_expr); + } + } + } - AnyExprV clone(AnyExprV v) final { - switch (v->type) { + static AnyExprV clone(AnyExprV v) { + switch (v->kind) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); + case ast_braced_expression: return clone(v->as()); + case ast_braced_yield_result: return clone(v->as()); + case ast_artificial_aux_vertex: return clone(v->as()); case ast_tensor: return clone(v->as()); - case ast_typed_tuple: return clone(v->as()); + case ast_bracket_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); case ast_local_var_lhs: return clone(v->as()); case ast_local_vars_declaration: return clone(v->as()); @@ -206,56 +301,72 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_binary_operator: return clone(v->as()); case ast_ternary_operator: return clone(v->as()); case ast_cast_as_operator: return clone(v->as()); + case ast_is_type_operator: return clone(v->as()); case ast_not_null_operator: return clone(v->as()); - case ast_is_null_check: return clone(v->as()); + case ast_lazy_operator: return clone(v->as()); + case ast_match_expression: return clone(v->as()); + case ast_match_arm: return clone(v->as()); + case ast_object_field: return clone(v->as()); + case ast_object_body: return clone(v->as()); + case ast_object_literal: return clone(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone(AnyExprV)"); } } - AnyV clone(AnyV v) final { - switch (v->type) { - case ast_empty_statement: return clone(v->as()); - case ast_sequence: return clone(v->as()); - case ast_return_statement: return clone(v->as()); - case ast_if_statement: return clone(v->as()); - case ast_repeat_statement: return clone(v->as()); - case ast_while_statement: return clone(v->as()); - case ast_do_while_statement: return clone(v->as()); - case ast_throw_statement: return clone(v->as()); - case ast_assert_statement: return clone(v->as()); - case ast_try_catch_statement: return clone(v->as()); - case ast_asm_body: return clone(v->as()); - // other AST nodes that can be children of ast nodes of function body - case ast_identifier: return clone(v->as()); - case ast_instantiationT_item: return clone(v->as()); - case ast_instantiationT_list: return clone(v->as()); - case ast_parameter: return clone(v->as()); - case ast_parameter_list: return clone(v->as()); - - default: { - // be very careful, don't forget to handle all statements/other (not expressions) above! - AnyExprV as_expr = reinterpret_cast(v); - return clone(as_expr); - } + static AnyTypeV clone(AnyTypeV v) { + if (v == nullptr) { + return nullptr; + } + switch (v->kind) { + case ast_type_leaf_text: return clone(v->as()); + case ast_type_question_nullable: return clone(v->as()); + case ast_type_parenthesis_tensor: return clone(v->as()); + case ast_type_bracket_tuple: return clone(v->as()); + case ast_type_arrow_callable: return clone(v->as()); + case ast_type_vertical_bar_union: return clone(v->as()); + case ast_type_triangle_args: return clone(v->as()); + default: + throw UnexpectedASTNodeKind(v, "ASTReplicator::clone(AnyTypeV)"); } } - TypePtr clone(TypePtr t) override { - return t; +public: + // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_function_ast(V v_orig) { + return createV( + v_orig->loc, + clone(v_orig->get_identifier()), + clone(v_orig->get_param_list()), + clone(v_orig->get_body()), + clone(v_orig->receiver_type_node), + clone(v_orig->return_type_node), + v_orig->genericsT_list ? clone(v_orig->genericsT_list) : nullptr, + v_orig->tvm_method_id, + v_orig->flags, + v_orig->inline_mode + ); } - public: - virtual V clone_function_body(V v_function) { - return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()->as()), - clone(v_function->declared_return_type), - v_function->genericsT_list, - v_function->method_id, - v_function->flags + // the cloned struct becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_struct_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + v_orig->overflow1023_policy, + v_orig->has_opcode() ? static_cast(clone(v_orig->get_opcode())) : createV(v_orig->loc), + clone(v_orig->get_struct_body()) + ); + } + + // the cloned type alias becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_type_alias_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + clone(v_orig->underlying_type_node) ); } }; diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index a7f260de8..fd19401e9 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -31,13 +31,24 @@ namespace tolk { class ASTStringifier final : public ASTVisitor { - constexpr static std::pair name_pairs[] = { + constexpr static std::pair name_pairs[] = { {ast_identifier, "ast_identifier"}, + // types + {ast_type_leaf_text, "ast_type_leaf_text"}, + {ast_type_question_nullable, "ast_type_question_nullable"}, + {ast_type_parenthesis_tensor, "ast_type_parenthesis_tensor"}, + {ast_type_bracket_tuple, "ast_type_bracket_tuple"}, + {ast_type_arrow_callable, "ast_type_arrow_callable"}, + {ast_type_vertical_bar_union, "ast_type_vertical_bar_union"}, + {ast_type_triangle_args, "ast_type_triangle_args"}, // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, + {ast_braced_expression, "ast_braced_expression"}, + {ast_braced_yield_result, "ast_braced_yield_result"}, + {ast_artificial_aux_vertex, "ast_artificial_aux_vertex"}, {ast_tensor, "ast_tensor"}, - {ast_typed_tuple, "ast_typed_tuple"}, + {ast_bracket_tuple, "ast_bracket_tuple"}, {ast_reference, "ast_reference"}, {ast_local_var_lhs, "ast_local_var_lhs"}, {ast_local_vars_declaration, "ast_local_vars_declaration"}, @@ -56,11 +67,17 @@ class ASTStringifier final : public ASTVisitor { {ast_binary_operator, "ast_binary_operator"}, {ast_ternary_operator, "ast_ternary_operator"}, {ast_cast_as_operator, "ast_cast_as_operator"}, + {ast_is_type_operator, "ast_is_type_operator"}, {ast_not_null_operator, "ast_not_null_operator"}, - {ast_is_null_check, "ast_is_null_check"}, + {ast_lazy_operator, "ast_lazy_operator"}, + {ast_match_expression, "ast_match_expression"}, + {ast_match_arm, "ast_match_arm"}, + {ast_object_field, "ast_object_field"}, + {ast_object_body, "ast_object_body"}, + {ast_object_literal, "ast_object_literal"}, // statements {ast_empty_statement, "ast_empty_statement"}, - {ast_sequence, "ast_sequence"}, + {ast_block_statement, "ast_block_statement"}, {ast_return_statement, "ast_return_statement"}, {ast_if_statement, "ast_if_statement"}, {ast_repeat_statement, "ast_repeat_statement"}, @@ -81,6 +98,10 @@ class ASTStringifier final : public ASTVisitor { {ast_function_declaration, "ast_function_declaration"}, {ast_global_var_declaration, "ast_global_var_declaration"}, {ast_constant_declaration, "ast_constant_declaration"}, + {ast_type_alias_declaration, "ast_type_alias_declaration"}, + {ast_struct_field, "ast_struct_field"}, + {ast_struct_body, "ast_struct_body"}, + {ast_struct_declaration, "ast_struct_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -88,29 +109,19 @@ class ASTStringifier final : public ASTVisitor { static_assert(std::size(name_pairs) == ast_tolk_file + 1, "name_pairs needs to be updated"); - constexpr static std::pair annotation_kinds[] = { - {AnnotationKind::inline_simple, "@inline"}, - {AnnotationKind::inline_ref, "@inline_ref"}, - {AnnotationKind::method_id, "@method_id"}, - {AnnotationKind::pure, "@pure"}, - {AnnotationKind::deprecated, "@deprecated"}, - }; - - static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); - - template - constexpr static const char* ast_node_type_to_string() { - return name_pairs[node_type].second; + template + constexpr static const char* ast_node_kind_to_string() { + return name_pairs[node_kind].second; } int depth = 0; std::string out; bool colored = false; - template - void handle_vertex(V v) { + template + void handle_vertex(V v) { out += std::string(depth * 2, ' '); - out += ast_node_type_to_string(); + out += ast_node_kind_to_string(); if (std::string postfix = specific_str(v); !postfix.empty()) { out += colored ? " \x1b[34m" : " // "; out += postfix; @@ -123,7 +134,9 @@ class ASTStringifier final : public ASTVisitor { } static std::string specific_str(AnyV v) { - switch (v->type) { + switch (v->kind) { + case ast_type_leaf_text: + return static_cast(v->as()->text); case ast_identifier: return static_cast(v->as()->name); case ast_reference: { @@ -136,11 +149,7 @@ class ASTStringifier final : public ASTVisitor { case ast_int_const: return static_cast(v->as()->orig_str); case ast_string_const: - if (char modifier = v->as()->modifier) { - return "\"" + static_cast(v->as()->str_val) + "\"" + std::string(1, modifier); - } else { - return "\"" + static_cast(v->as()->str_val) + "\""; - } + return "\"" + static_cast(v->as()->str_val) + "\""; case ast_bool_const: return v->as()->bool_val ? "true" : "false"; case ast_dot_access: { @@ -161,6 +170,12 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name); case ast_constant_declaration: return static_cast(v->as()->get_identifier()->name); + case ast_type_alias_declaration: + return "type " + static_cast(v->as()->get_identifier()->name); + case ast_struct_field: + return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); + case ast_struct_declaration: + return "struct " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -170,20 +185,21 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return static_cast(v->as()->operator_name); case ast_cast_as_operator: - return v->as()->cast_to_type->as_human_readable(); - case ast_sequence: - return "↓" + std::to_string(v->as()->get_items().size()); + return ast_type_node_to_string(v->as()->type_node); + case ast_is_type_operator: { + std::string prefix = v->as()->is_negated ? "!is " : "is "; + return prefix + ast_type_node_to_string(v->as()->type_node); + } + case ast_block_statement: + return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: - return v->as()->substituted_type->as_human_readable(); + return ast_type_node_to_string(v->as()->type_node); case ast_if_statement: return v->as()->is_ifnot ? "ifnot" : ""; case ast_annotation: - return annotation_kinds[static_cast(v->as()->kind)].second; - case ast_parameter: { - std::ostringstream os; - os << v->as()->declared_type; - return static_cast(v->as()->param_name) + ": " + os.str(); - } + return static_cast(v->as()->name); + case ast_parameter: + return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { std::string param_names; for (int i = 0; i < v->as()->get_num_params(); i++) { @@ -191,31 +207,47 @@ class ASTStringifier final : public ASTVisitor { param_names += ","; param_names += v->as()->get_param(i)->param_name; } - return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; + std::string decl = "fun "; + if (auto receiver_node = v->as()->receiver_type_node) { + decl += specific_str(receiver_node); + decl += "."; + } + return decl + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; } case ast_local_var_lhs: { - std::ostringstream os; - os << (v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : v->as()->declared_type->as_human_readable()); + std::string str_type = v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : ast_type_node_to_string(v->as()->type_node); if (v->as()->get_name().empty()) { - return "_: " + os.str(); + return "_: " + str_type; } - return static_cast(v->as()->get_name()) + ":" + os.str(); + return static_cast(v->as()->get_name()) + ": " + str_type; } case ast_instantiationT_list: { std::string result = "<"; for (AnyV item : v->as()->get_items()) { if (result.size() > 1) result += ","; - result += item->as()->substituted_type->as_human_readable(); + result += ast_type_node_to_string(item->as()->type_node); } return result + ">"; } + case ast_match_arm: + if (v->as()->pattern_kind == MatchArmKind::exact_type) { + return ast_type_node_to_string(v->as()->pattern_type_node); + } + if (v->as()->pattern_kind == MatchArmKind::const_expression) { + return "(expression)"; + } + return "(else)"; + case ast_object_field: + return static_cast(v->as()->get_field_name()); + case ast_object_literal: + return "↓" + std::to_string(v->as()->get_body()->get_num_fields()); case ast_tolk_required_version: return static_cast(v->as()->semver); case ast_import_directive: return static_cast(v->as()->get_file_leaf()->str_val); case ast_tolk_file: - return v->as()->file->rel_filename; + return v->as()->file->realpath; default: return {}; } @@ -232,7 +264,7 @@ class ASTStringifier final : public ASTVisitor { } static std::string to_string_without_children(AnyV v) { - std::string result = ast_node_type_to_string(v->type); + std::string result = ast_node_kind_to_string(v->kind); if (std::string postfix = specific_str(v); !postfix.empty()) { result += ' '; result += specific_str(v); @@ -240,18 +272,42 @@ class ASTStringifier final : public ASTVisitor { return result; } - static const char* ast_node_type_to_string(ASTNodeType node_type) { - return name_pairs[node_type].second; + static std::string ast_type_node_to_string(AnyTypeV type_node) { + if (type_node == nullptr) { + return ""; + } + if (auto v_leaf = type_node->try_as()) { + return static_cast(v_leaf->text); + } + if (auto v_nullable = type_node->try_as()) { + return ast_type_node_to_string(v_nullable->get_inner()) + "?"; + } + return ast_node_kind_to_string(type_node->kind); + } + + static const char* ast_node_kind_to_string(ASTNodeKind node_kind) { + return name_pairs[node_kind].second; } void visit(AnyV v) override { - switch (v->type) { + switch (v->kind) { case ast_identifier: return handle_vertex(v->as()); + // types + case ast_type_leaf_text: return handle_vertex(v->as()); + case ast_type_question_nullable: return handle_vertex(v->as()); + case ast_type_parenthesis_tensor: return handle_vertex(v->as()); + case ast_type_bracket_tuple: return handle_vertex(v->as()); + case ast_type_arrow_callable: return handle_vertex(v->as()); + case ast_type_vertical_bar_union: return handle_vertex(v->as()); + case ast_type_triangle_args: return handle_vertex(v->as()); // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); + case ast_braced_expression: return handle_vertex(v->as()); + case ast_braced_yield_result: return handle_vertex(v->as()); + case ast_artificial_aux_vertex: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); - case ast_typed_tuple: return handle_vertex(v->as()); + case ast_bracket_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); case ast_local_var_lhs: return handle_vertex(v->as()); case ast_local_vars_declaration: return handle_vertex(v->as()); @@ -270,11 +326,17 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return handle_vertex(v->as()); case ast_ternary_operator: return handle_vertex(v->as()); case ast_cast_as_operator: return handle_vertex(v->as()); + case ast_is_type_operator: return handle_vertex(v->as()); case ast_not_null_operator: return handle_vertex(v->as()); - case ast_is_null_check: return handle_vertex(v->as()); + case ast_lazy_operator: return handle_vertex(v->as()); + case ast_match_expression: return handle_vertex(v->as()); + case ast_match_arm: return handle_vertex(v->as()); + case ast_object_field: return handle_vertex(v->as()); + case ast_object_body: return handle_vertex(v->as()); + case ast_object_literal: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); - case ast_sequence: return handle_vertex(v->as()); + case ast_block_statement: return handle_vertex(v->as()); case ast_return_statement: return handle_vertex(v->as()); case ast_if_statement: return handle_vertex(v->as()); case ast_repeat_statement: return handle_vertex(v->as()); @@ -295,11 +357,15 @@ class ASTStringifier final : public ASTVisitor { case ast_function_declaration: return handle_vertex(v->as()); case ast_global_var_declaration: return handle_vertex(v->as()); case ast_constant_declaration: return handle_vertex(v->as()); + case ast_type_alias_declaration: return handle_vertex(v->as()); + case ast_struct_field: return handle_vertex(v->as()); + case ast_struct_body: return handle_vertex(v->as()); + case ast_struct_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTStringifier::visit"); + throw UnexpectedASTNodeKind(v, "ASTStringifier::visit"); } } }; diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index d697aa82b..6e38ba792 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -21,7 +21,7 @@ /* * A module implementing base functionality of read-only traversing a vertex tree. - * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_type. + * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_kind. * E.g., for ast_if_statement, visit nodes cond, if-body and else-body. For ast_string_const, nothing. And so on. * Visitors below are helpers to inherit from and handle specific vertex types. * @@ -37,6 +37,16 @@ namespace tolk { class ASTVisitor { protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTTypeLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTTypeVararg* v) { + for (AnyTypeV child : v->children) { + visit(child); + } + } + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTExprLeaf* v) { static_cast(v); } @@ -56,6 +66,10 @@ class ASTVisitor { } } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { + visit(v->child_block_statement->as()); + } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { visit(v->child); } @@ -89,8 +103,11 @@ class ASTVisitorFunctionBody : public ASTVisitor { // expressions virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -109,11 +126,17 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -124,12 +147,15 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } void visit(AnyV v) final { - switch (v->type) { + switch (v->kind) { // expressions case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); + case ast_braced_expression: return visit(v->as()); + case ast_braced_yield_result: return visit(v->as()); + case ast_artificial_aux_vertex: return visit(v->as()); case ast_tensor: return visit(v->as()); - case ast_typed_tuple: return visit(v->as()); + case ast_bracket_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); case ast_local_var_lhs: return visit(v->as()); case ast_local_vars_declaration: return visit(v->as()); @@ -148,11 +174,17 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_binary_operator: return visit(v->as()); case ast_ternary_operator: return visit(v->as()); case ast_cast_as_operator: return visit(v->as()); + case ast_is_type_operator: return visit(v->as()); case ast_not_null_operator: return visit(v->as()); - case ast_is_null_check: return visit(v->as()); + case ast_lazy_operator: return visit(v->as()); + case ast_match_expression: return visit(v->as()); + case ast_match_arm: return visit(v->as()); + case ast_object_field: return visit(v->as()); + case ast_object_body: return visit(v->as()); + case ast_object_literal: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); - case ast_sequence: return visit(v->as()); + case ast_block_statement: return visit(v->as()); case ast_return_statement: return visit(v->as()); case ast_if_statement: return visit(v->as()); case ast_repeat_statement: return visit(v->as()); @@ -163,10 +195,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_try_catch_statement: return visit(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); + throw UnexpectedASTNodeKind(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); #endif default: - throw UnexpectedASTNodeType(v, "ASTVisitorFunctionBody::visit"); + throw UnexpectedASTNodeKind(v, "ASTVisitorFunctionBody::visit"); } } @@ -180,11 +212,15 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void visit_ast_of_all_functions() { BodyVisitorT visitor; - for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + const std::vector& all = get_all_not_builtin_functions(); + for (size_t i = 0; i < all.size(); ++i) { // NOLINT(*-loop-convert) + FunctionPtr fun_ref = all[i]; // not range-base loop to prevent iterator invalidation (push_back at generics) if (visitor.should_visit_function(fun_ref)) { visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); } diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 26eaacd59..b22f9d31f 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -36,10 +36,10 @@ void ASTNodeBase::debug_print() const { #endif // TOLK_DEBUG -UnexpectedASTNodeType::UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { - message = "Unexpected ASTNodeType "; +UnexpectedASTNodeKind::UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { + message = "Unexpected ASTNodeKind "; #ifdef TOLK_DEBUG - message += ASTStringifier::ast_node_type_to_string(v_unexpected->type); + message += ASTStringifier::ast_node_kind_to_string(v_unexpected->kind); message += " "; #endif message += "in "; @@ -60,12 +60,24 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@inline_ref") { return AnnotationKind::inline_ref; } + if (name == "@noinline") { + return AnnotationKind::noinline; + } if (name == "@method_id") { return AnnotationKind::method_id; } if (name == "@deprecated") { return AnnotationKind::deprecated; } + if (name == "@custom") { + return AnnotationKind::custom; + } + if (name == "@overflow1023_policy") { + return AnnotationKind::overflow1023_policy; + } + if (name == "@on_bounced_policy") { + return AnnotationKind::on_bounced_policy; + } return AnnotationKind::unknown; } @@ -105,6 +117,10 @@ int Vertex::get_mutate_params_count() const { // Therefore, there is a guarantee, that all AST mutations are done via these methods, // easily searched by usages, and there is no another way to modify any other field. +void ASTNodeDeclaredTypeBase::assign_resolved_type(TypePtr resolved_type) { + this->resolved_type = resolved_type; +} + void ASTNodeExpressionBase::assign_inferred_type(TypePtr type) { this->inferred_type = type; } @@ -126,40 +142,43 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } -void Vertex::assign_fun_ref(FunctionPtr fun_ref) { - this->fun_maybe = fun_ref; +void Vertex::assign_literal_value(std::string&& literal_value) { + this->literal_value = std::move(literal_value); } -void Vertex::assign_resolved_type(TypePtr cast_to_type) { - this->cast_to_type = cast_to_type; +void Vertex::assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self) { + this->fun_maybe = fun_ref; + this->dot_obj_is_self = dot_obj_is_self; } -void Vertex::assign_var_ref(GlobalVarPtr var_ref) { - this->var_ref = var_ref; +void Vertex::assign_is_negated(bool is_negated) { + this->is_negated = is_negated; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_dest_var_ref(LocalVarPtr dest_var_ref) { + this->dest_var_ref = dest_var_ref; } -void Vertex::assign_const_ref(GlobalConstPtr const_ref) { - this->const_ref = const_ref; +void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { + this->pattern_type_node = nullptr; + this->pattern_kind = pattern_kind; + this->lhs = pattern_expr; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_glob_ref(GlobalVarPtr glob_ref) { + this->glob_ref = glob_ref; } -void Vertex::assign_resolved_type(TypePtr substituted_type) { - this->substituted_type = substituted_type; +void Vertex::assign_const_ref(GlobalConstPtr const_ref) { + this->const_ref = const_ref; } -void Vertex::assign_param_ref(LocalVarPtr param_ref) { - this->param_ref = param_ref; +void Vertex::assign_alias_ref(AliasDefPtr alias_ref) { + this->alias_ref = alias_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; } void Vertex::assign_fun_ref(FunctionPtr fun_ref) { @@ -174,32 +193,32 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_is_negated(bool is_negated) { - this->is_negated = is_negated; +void Vertex::assign_first_unreachable(AnyV first_unreachable) { + this->first_unreachable = first_unreachable; } -void Vertex::assign_first_unreachable(AnyV first_unreachable) { - this->first_unreachable = first_unreachable; +void Vertex::assign_new_children(std::vector&& children) { + this->children = std::move(children); } void Vertex::assign_target(const DotTarget& target) { this->target = target; } -void Vertex::assign_fun_ref(FunctionPtr fun_ref) { - this->fun_ref = fun_ref; +void Vertex::assign_field_ref(StructFieldPtr field_ref) { + this->field_ref = field_ref; } -void Vertex::assign_resolved_type(TypePtr declared_return_type) { - this->declared_return_type = declared_return_type; +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; } -void Vertex::assign_var_ref(LocalVarPtr var_ref) { - this->var_ref = var_ref; +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_var_ref(LocalVarPtr var_ref) { + this->var_ref = var_ref; } void Vertex::assign_src_file(const SrcFile* file) { diff --git a/tolk/ast.h b/tolk/ast.h index 9b7c5d1a5..8836d2a06 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -27,35 +27,33 @@ * Here we introduce AST representation of Tolk source code. * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. * There was no way to perform any more or less semantic analysis. - * In Tolk, I've implemented parsing .tolk files into AST at first, and then converting this AST - * into legacy representation (see pipe-ast-to-legacy.cpp). - * In the future, more and more code analysis will be moved out of legacy to AST-level. + * In Tolk, all files are parsed into AST, and all semantic analysis is done at the AST level. * * From the user's point of view, all AST vertices are constant. All API is based on constancy. * Even though fields of vertex structs are public, they can't be modified, since vertices are accepted by const ref. * Generally, there are three ways of accepting a vertex: * * AnyV (= const ASTNodeBase*) - * the only you can do with this vertex is to see v->type (ASTNodeType) and to cast via v->as() + * the only you can do with this vertex is to see v->kind (ASTNodeKind) and to cast via v->as() * * AnyExprV (= const ASTNodeExpressionBase*) * in contains expression-specific properties (lvalue/rvalue, inferred type) - * * V (= const Vertex*) + * * V (= const Vertex*) * a specific type of vertex, you can use its fields and methods * There is one way of creating a vertex: - * * createV(...constructor_args) (= new Vertex(...)) + * * createV(...constructor_args) (= new Vertex(...)) * vertices are currently created on a heap, without any custom memory arena, just allocated and never deleted * The only way to modify a field is to use "mutate()" method (drops constancy, the only point of mutation) - * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. + * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. * - * Having AnyV and knowing its node_type, a call - * v->as() + * Having AnyV and knowing its node_kind, a call + * v->as() * will return a typed vertex. - * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: + * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: * if (auto v_int = v->try_as()) * Note, that there casts are NOT DYNAMIC. ASTNode is not a virtual base, it has no vtable. * So, as<...>() is just a compile-time casting, without any runtime overhead. * * Note, that ASTNodeBase doesn't store any vector of children. That's why there is no way to loop over - * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). + * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). * Hence, to iterate over a custom vertex (e.g., a function body), one should inherit some kind of ASTVisitor. * Besides read-only visiting, there is a "visit and replace" pattern. * See ast-visitor.h and ast-replacer.h. @@ -63,13 +61,24 @@ namespace tolk { -enum ASTNodeType { +enum ASTNodeKind { ast_identifier, + // types + ast_type_leaf_text, + ast_type_question_nullable, + ast_type_parenthesis_tensor, + ast_type_bracket_tuple, + ast_type_arrow_callable, + ast_type_vertical_bar_union, + ast_type_triangle_args, // expressions ast_empty_expression, ast_parenthesized_expression, + ast_braced_expression, + ast_braced_yield_result, + ast_artificial_aux_vertex, ast_tensor, - ast_typed_tuple, + ast_bracket_tuple, ast_reference, ast_local_var_lhs, ast_local_vars_declaration, @@ -88,11 +97,17 @@ enum ASTNodeType { ast_binary_operator, ast_ternary_operator, ast_cast_as_operator, + ast_is_type_operator, ast_not_null_operator, - ast_is_null_check, + ast_lazy_operator, + ast_match_expression, + ast_match_arm, + ast_object_field, + ast_object_body, + ast_object_literal, // statements ast_empty_statement, - ast_sequence, + ast_block_statement, ast_return_statement, ast_if_statement, ast_repeat_statement, @@ -113,6 +128,10 @@ enum ASTNodeType { ast_function_declaration, ast_global_var_declaration, ast_constant_declaration, + ast_type_alias_declaration, + ast_struct_field, + ast_struct_body, + ast_struct_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -121,25 +140,39 @@ enum ASTNodeType { enum class AnnotationKind { inline_simple, inline_ref, + noinline, method_id, pure, deprecated, + custom, + overflow1023_policy, + on_bounced_policy, unknown, }; -template +enum class MatchArmKind { // for `match` expression, each of arms `pattern => body` can be: + const_expression, // `-1 => body` / `SOME_CONST + ton("0.05") => body` (any expr at parsing, resulting in const) + exact_type, // `int => body` / `User | slice => body` + else_branch, // `else => body` +}; + +struct ASTAuxData { // base class for data in ast_artificial_aux_vertex, see ast-aux-data.h + virtual ~ASTAuxData() = default; +}; + +template struct Vertex; -template -using V = const Vertex*; +template +using V = const Vertex*; #define createV new Vertex -struct UnexpectedASTNodeType final : std::exception { +struct UnexpectedASTNodeKind final : std::exception { AnyV v_unexpected; std::string message; - explicit UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where); + explicit UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where); const char* what() const noexcept override { return message.c_str(); @@ -149,25 +182,25 @@ struct UnexpectedASTNodeType final : std::exception { // --------------------------------------------------------- struct ASTNodeBase { - const ASTNodeType type; + const ASTNodeKind kind; const SrcLocation loc; - ASTNodeBase(ASTNodeType type, SrcLocation loc) : type(type), loc(loc) {} + ASTNodeBase(ASTNodeKind kind, SrcLocation loc) : kind(kind), loc(loc) {} ASTNodeBase(const ASTNodeBase&) = delete; - template - V as() const { + template + V as() const { #ifdef TOLK_DEBUG - if (type != node_type) { - throw Fatal("v->as<...> to wrong node_type"); + if (kind != node_kind) { + throw Fatal("v->as<...> to wrong node_kind"); } #endif - return static_cast>(this); + return static_cast>(this); } - template - V try_as() const { - return type == node_type ? static_cast>(this) : nullptr; + template + V try_as() const { + return kind == node_kind ? static_cast>(this) : nullptr; } #ifdef TOLK_DEBUG @@ -180,9 +213,16 @@ struct ASTNodeBase { void error(const std::string& err_msg) const; }; -struct ASTNodeExpressionBase : ASTNodeBase { - friend class ASTDuplicatorFunction; +struct ASTNodeDeclaredTypeBase : ASTNodeBase { + TypePtr resolved_type = nullptr; + ASTNodeDeclaredTypeBase* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr resolved_type); + + ASTNodeDeclaredTypeBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; + +struct ASTNodeExpressionBase : ASTNodeBase { TypePtr inferred_type = nullptr; bool is_rvalue: 1 = false; bool is_lvalue: 1 = false; @@ -195,11 +235,27 @@ struct ASTNodeExpressionBase : ASTNodeBase { void assign_lvalue_true(); void assign_always_true_or_false(int flow_true_false_state); - ASTNodeExpressionBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeExpressionBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} }; struct ASTNodeStatementBase : ASTNodeBase { - ASTNodeStatementBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeStatementBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; + +struct ASTTypeLeaf : ASTNodeDeclaredTypeBase { +protected: + ASTTypeLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeDeclaredTypeBase(kind, loc) {} +}; + +struct ASTTypeVararg : ASTNodeDeclaredTypeBase { + friend class ASTVisitor; + +protected: + std::vector children; + + ASTTypeVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeDeclaredTypeBase(kind, loc), children(std::move(children)) {} }; struct ASTExprLeaf : ASTNodeExpressionBase { @@ -207,8 +263,8 @@ struct ASTExprLeaf : ASTNodeExpressionBase { friend class ASTReplacer; protected: - ASTExprLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeExpressionBase(type, loc) {} + ASTExprLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeExpressionBase(kind, loc) {} }; struct ASTExprUnary : ASTNodeExpressionBase { @@ -218,8 +274,8 @@ struct ASTExprUnary : ASTNodeExpressionBase { protected: AnyExprV child; - ASTExprUnary(ASTNodeType type, SrcLocation loc, AnyExprV child) - : ASTNodeExpressionBase(type, loc), child(child) {} + ASTExprUnary(ASTNodeKind kind, SrcLocation loc, AnyExprV child) + : ASTNodeExpressionBase(kind, loc), child(child) {} }; struct ASTExprBinary : ASTNodeExpressionBase { @@ -230,8 +286,8 @@ struct ASTExprBinary : ASTNodeExpressionBase { AnyExprV lhs; AnyExprV rhs; - ASTExprBinary(ASTNodeType type, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) - : ASTNodeExpressionBase(type, loc), lhs(lhs), rhs(rhs) {} + ASTExprBinary(ASTNodeKind kind, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(kind, loc), lhs(lhs), rhs(rhs) {} }; struct ASTExprVararg : ASTNodeExpressionBase { @@ -243,14 +299,25 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} + ASTExprVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeExpressionBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } bool empty() const { return children.empty(); } }; +struct ASTExprBlockOfStatements : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyV child_block_statement; + + ASTExprBlockOfStatements(ASTNodeKind kind, SrcLocation loc, AnyV child_block_statement) + : ASTNodeExpressionBase(kind, loc), child_block_statement(child_block_statement) {} +}; + struct ASTStatementUnary : ASTNodeStatementBase { friend class ASTVisitor; friend class ASTReplacer; @@ -260,8 +327,8 @@ struct ASTStatementUnary : ASTNodeStatementBase { AnyExprV child_as_expr() const { return reinterpret_cast(child); } - ASTStatementUnary(ASTNodeType type, SrcLocation loc, AnyV child) - : ASTNodeStatementBase(type, loc), child(child) {} + ASTStatementUnary(ASTNodeKind kind, SrcLocation loc, AnyV child) + : ASTNodeStatementBase(kind, loc), child(child) {} }; struct ASTStatementVararg : ASTNodeStatementBase { @@ -273,8 +340,8 @@ struct ASTStatementVararg : ASTNodeStatementBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTStatementVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeStatementBase(type, loc), children(std::move(children)) {} + ASTStatementVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeStatementBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -286,8 +353,8 @@ struct ASTOtherLeaf : ASTNodeBase { friend class ASTReplacer; protected: - ASTOtherLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeBase(type, loc) {} + ASTOtherLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeBase(kind, loc) {} }; struct ASTOtherVararg : ASTNodeBase { @@ -299,8 +366,8 @@ struct ASTOtherVararg : ASTNodeBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTOtherVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeBase(type, loc), children(std::move(children)) {} + ASTOtherVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -325,6 +392,86 @@ struct Vertex final : ASTOtherLeaf { }; +// +// --------------------------------------------------------- +// types +// + + +template<> +// ast_type_leaf_text is a type node without children: "int", "User", "T", etc. +// after resolving, it will become TypeDataInt / TypeDataStruct / etc. +struct Vertex final : ASTTypeLeaf { + std::string_view text; + + Vertex(SrcLocation loc, std::string_view text) + : ASTTypeLeaf(ast_type_leaf_text, loc) + , text(text) {} +}; + +template<> +// ast_type_question_nullable is "T?" +// after resolving, it will become a union "T | null", but at AST level, it's a separate node +struct Vertex final : ASTTypeVararg { + AnyTypeV get_inner() const { return children.at(0); } + + Vertex(SrcLocation loc, AnyTypeV inner) + : ASTTypeVararg(ast_type_question_nullable, loc, {inner}) {} +}; + +template<> +// ast_type_parenthesis_tensor is "(T1, T2, ...)" +// after resolving, it will become TypeDataTensor +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_parenthesis_tensor, loc, std::move(items)) {} +}; + +template<> +// ast_type_bracket_tuple is "[T1, T2, ...]" +// after resolving, it will become TypeDataBrackets +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_bracket_tuple, loc, std::move(items)) {} +}; + +template<> +// ast_type_arrow_callable is "(T1, T2, ...) -> TResult" +// after resolving, it will become TypeDataFunCallable +struct Vertex final : ASTTypeVararg { + const std::vector& get_params_and_return() const { return children; } + + Vertex(SrcLocation loc, std::vector&& params_and_return) + : ASTTypeVararg(ast_type_arrow_callable, loc, std::move(params_and_return)) {} +}; + +template<> +// ast_type_vertical_bar_union is "T1 | T2 | ..." +// after resolving, it will become TypeDataUnion +struct Vertex final : ASTTypeVararg { + const std::vector& get_variants() const { return children; } + + Vertex(SrcLocation loc, std::vector&& variants) + : ASTTypeVararg(ast_type_vertical_bar_union, loc, std::move(variants)) {} +}; + + +template<> +// ast_type_triangle_args is "T1" +// example: `type OkInt = Ok`, then "Ok" is triangle args, and at resolving, it's instantiated into TypeDataStruct +// example: `type A = Ok`, then "Ok" is triangle args, but at resolving, kept as TypeDataGenericTypeWithTs +struct Vertex final : ASTTypeVararg { + const std::vector& get_inner_and_args() const { return children; } + + Vertex(SrcLocation loc, std::vector&& inner_and_args) + : ASTTypeVararg(ast_type_triangle_args, loc, std::move(inner_and_args)) {} +}; + + // // --------------------------------------------------------- // expressions @@ -350,6 +497,45 @@ struct Vertex final : ASTExprUnary { : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} }; +template<> +// ast_braced_expression is a sequence, but in a context of expression (it has a type) +// it can contain arbitrary statements inside +// it can occur only in special places within the input code, not anywhere +// example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression +// example: `match (intV) { 0 => 1 }` rhs is implicitly wrapped to a braced expression with `1` yielded +struct Vertex final : ASTExprBlockOfStatements { + auto get_block_statement() const { return child_block_statement->as(); } + + Vertex(SrcLocation loc, AnyV child_block_statement) + : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} +}; + +template<> +// ast_braced_yield_result is a special vertex to return a value from a braced expression +// example: `match (intV) { 0 => 1 }` rhs of 0 is implicitly wrapped to a braced expression with `1` yielded +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_braced_yield_result, loc, expr) {} +}; + +template<> +// ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code +// example: special vertex to force location in Fift output at constant usage +// example: `msg.isBounced` / `msg.xxx` in onInternalMessage are handled specially +struct Vertex final : ASTExprUnary { + const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h + + AnyExprV get_wrapped_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV wrapped_expr, const ASTAuxData* aux_data, TypePtr inferred_type) + : ASTExprUnary(ast_artificial_aux_vertex, loc, wrapped_expr) + , aux_data(aux_data) { + assign_inferred_type(inferred_type); + } +}; + template<> // ast_tensor is a set of expressions embraced by (parenthesis) // in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" @@ -365,16 +551,16 @@ struct Vertex final : ASTExprVararg { }; template<> -// ast_typed_tuple is a set of expressions in [square brackets] +// ast_bracket_tuple is a set of expressions in [square brackets] // in TVM, it's a TVM tuple, that occupies 1 slot, but the compiler knows its "typed structure" // example: `[1, x]`, `[[0]]` (nested) // typed tuples can be assigned to N variables, like `[one, _, three] = [1,2,3]` -struct Vertex final : ASTExprVararg { +struct Vertex final : ASTExprVararg { const std::vector& get_items() const { return children; } AnyExprV get_item(int i) const { return child(i); } Vertex(SrcLocation loc, std::vector items) - : ASTExprVararg(ast_typed_tuple, loc, std::move(items)) {} + : ASTExprVararg(ast_bracket_tuple, loc, std::move(items)) {} }; template<> @@ -414,8 +600,9 @@ struct Vertex final : ASTExprLeaf { public: LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty - TypePtr declared_type; // not null for `var x: int = rhs`, otherwise nullptr + AnyTypeV type_node; // exists for `var x: int = rhs`, otherwise nullptr bool is_immutable; // declared via 'val', not 'var' + bool is_lateinit; // var st: Storage lateinit (no assignment) bool marked_as_redef; // var (existing_var redef, new_var: int) = ... V get_identifier() const { return identifier; } @@ -423,11 +610,10 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V identifier, TypePtr declared_type, bool is_immutable, bool marked_as_redef) + Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool is_lateinit, bool marked_as_redef) : ASTExprLeaf(ast_local_var_lhs, loc) - , identifier(identifier), declared_type(declared_type), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} + , identifier(identifier), type_node(type_node), is_immutable(is_immutable), is_lateinit(is_lateinit), marked_as_redef(marked_as_redef) {} }; template<> @@ -436,7 +622,7 @@ template<> // for `var (x, [y])` its expr is "tensor (local var, typed tuple (local var))" // for assignment `var x = 5`, this node is `var x`, lhs of assignment struct Vertex final : ASTExprUnary { - AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_typed_tuple + AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_bracket_tuple Vertex(SrcLocation loc, AnyExprV expr) : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} @@ -458,25 +644,18 @@ struct Vertex final : ASTExprLeaf { template<> // ast_string_const is a string literal in double quotes or """ when multiline -// examples: "asdf" / "Ef8zMz..."a / "to_calc_crc32_from"c -// an optional modifier specifies how a string is parsed (probably, like an integer) +// examples: "asdf" / "LTIME" (in asm body) / stringCrc32("asdf") (as an argument) // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - char modifier; + std::string literal_value; // when "some_str" is a standalone string, value of type `slice`, for x{...} Fift output - bool is_bitslice() const { - char m = modifier; - return m == 0 || m == 's' || m == 'a'; - } - bool is_intval() const { - char m = modifier; - return m == 'u' || m == 'h' || m == 'H' || m == 'c'; - } + Vertex* mutate() const { return const_cast(this); } + void assign_literal_value(std::string&& literal_value); - Vertex(SrcLocation loc, std::string_view str_val, char modifier) + Vertex(SrcLocation loc, std::string_view str_val) : ASTExprLeaf(ast_string_const, loc) - , str_val(str_val), modifier(modifier) {} + , str_val(str_val) {} }; template<> @@ -524,7 +703,7 @@ struct Vertex final : ASTExprVararg { template<> // ast_dot_access is "object before dot, identifier + optional after dot" -// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` +// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` / `Point.create` // from traversing point of view, it's an unary expression: only obj is expression, field name is not // note, that `obj.method()` is a function call with "dot access `obj.method`" callee struct Vertex final : ASTExprUnary { @@ -536,11 +715,13 @@ struct Vertex final : ASTExprUnary { typedef std::variant< FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function + StructFieldPtr, // for `user.id` target is field `id` of struct `User` int // for `t.0` target is "indexed access" 0 > DotTarget; - DotTarget target = static_cast(nullptr); // filled at type inferring + DotTarget target = static_cast(nullptr); // filled at type inferring bool is_target_fun_ref() const { return std::holds_alternative(target); } + bool is_target_struct_field() const { return std::holds_alternative(target); } bool is_target_indexed_access() const { return std::holds_alternative(target); } AnyExprV get_obj() const { return child; } @@ -563,19 +744,20 @@ template<> // example: `globalF()` then callee is reference (with instantiation Ts filled) // example: `local_var()` then callee is reference (points to local var, filled at resolve identifiers) // example: `getF()()` then callee is another func call (which type is TypeDataFunCallable) -// example: `obj.method()` then callee is dot access (resolved while type inferring) +// example: `obj.method()` then callee is dot access, self_obj = obj +// example: `Point.create()` then callee is dot access, self_obj = nullptr struct Vertex final : ASTExprBinary { FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` + bool dot_obj_is_self = false; // true for `obj.method()` (obj will be `self` in method); false for `globalF()` / `Point.create()` AnyExprV get_callee() const { return lhs; } - bool is_dot_call() const { return lhs->type == ast_dot_access; } - AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } + AnyExprV get_self_obj() const { return dot_obj_is_self ? lhs->as()->get_obj() : nullptr; } auto get_arg_list() const { return rhs->as(); } int get_num_args() const { return rhs->as()->size(); } auto get_arg(int i) const { return rhs->as()->get_arg(i); } Vertex* mutate() const { return const_cast(this); } - void assign_fun_ref(FunctionPtr fun_ref); + void assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self); Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} @@ -677,16 +859,30 @@ template<> // ast_cast_as_operator is explicit casting with "as" keyword // examples: `arg as int` / `null as cell` / `t.tupleAt(2) as slice` struct Vertex final : ASTExprUnary { + AnyTypeV type_node; + + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node) + : ASTExprUnary(ast_cast_as_operator, loc, expr) + , type_node(type_node) {} +}; + +template<> +// ast_is_type_operator is type matching with "is" or "!is" keywords and "== null" / "!= null" (same as "is null") +// examples: `v is SomeStruct` / `getF() !is slice` / `v == null` / `v !is null` +struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - TypePtr cast_to_type; + AnyTypeV type_node; + bool is_negated; // `!is type`, `!= null` Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr cast_to_type); + void assign_is_negated(bool is_negated); - Vertex(SrcLocation loc, AnyExprV expr, TypePtr cast_to_type) - : ASTExprUnary(ast_cast_as_operator, loc, expr) - , cast_to_type(cast_to_type) {} + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node, bool is_negated) + : ASTExprUnary(ast_is_type_operator, loc, expr) + , type_node(type_node), is_negated(is_negated) {} }; template<> @@ -700,19 +896,108 @@ struct Vertex final : ASTExprUnary { }; template<> -// ast_is_null_check is an artificial vertex for "expr == null" / "expr != null" / same but null on the left -// it's created instead of a general binary expression to emphasize its purpose -struct Vertex final : ASTExprUnary { - bool is_negated; +// ast_lazy_operator is `lazy (loading-expr)` for lazy/partial loading +// example: `lazy Storage.fromCell(contract.getData())` +struct Vertex final : ASTExprUnary { + LocalVarPtr dest_var_ref = nullptr; // `var st = lazy Storage.load()` AnyExprV get_expr() const { return child; } Vertex* mutate() const { return const_cast(this); } - void assign_is_negated(bool is_negated); + void assign_dest_var_ref(LocalVarPtr dest_var_ref); - Vertex(SrcLocation loc, AnyExprV expr, bool is_negated) - : ASTExprUnary(ast_is_null_check, loc, expr) - , is_negated(is_negated) {} + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_lazy_operator, loc, expr) {} +}; + +template<> +// ast_match_expression is `match (subject) { ... arms ... }`, used either as a statement or as an expression +// example: `match (intOrSliceVar) { int => 1, slice => 2 }` +// example: `match (var c = getIntOrSlice()) { int => return 0, slice => throw 123 }` +struct Vertex final : ASTExprVararg { + AnyExprV get_subject() const { return child(0); } + int get_arms_count() const { return size() - 1; } + auto get_arm(int i) const { return child(i + 1)->as(); } + const std::vector& get_all_children() const { return children; } + + bool is_statement() const { return !is_rvalue && !is_lvalue; } + + Vertex(SrcLocation loc, std::vector&& subject_and_arms) + : ASTExprVararg(ast_match_expression, loc, std::move(subject_and_arms)) {} +}; + +template<> +// ast_match_arm is one `pattern => body` inside `match` expression/statement +// pattern can be a custom expression / a type / `else` (see comments in MatchArmKind) +// body can be any expression; particularly, braced expression `{ ... }` +// example: `int => variable` (match by type, inferred_type of body variable's type) +// example: `a+b => { ...; return 0; }` (match by expression, inferred_type of body is "never" (unreachable end)) +struct Vertex final : ASTExprBinary { + MatchArmKind pattern_kind; + AnyTypeV pattern_type_node; // for MatchArmKind::exact_type; otherwise nullptr + + AnyExprV get_pattern_expr() const { return lhs; } + auto get_body() const { return rhs->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); + + Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, V body) + : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) + , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} +}; + +template<> +// ast_object_field is one field at object creation +// example: `Point { x: 2, y: 3 }` is object creation, its body contains 2 fields +struct Vertex final : ASTExprUnary { +private: + V identifier; + +public: + StructFieldPtr field_ref = nullptr; // assigned at type inferring + + AnyExprV get_init_val() const { return child; } + std::string_view get_field_name() const { return identifier->name; } + V get_field_identifier() const { return identifier; } + + Vertex* mutate() const { return const_cast(this); } + void assign_field_ref(StructFieldPtr field_ref); + + Vertex(SrcLocation loc, V name_identifier, AnyExprV init_val) + : ASTExprUnary(ast_object_field, loc, init_val) + , identifier(name_identifier) {} +}; + +template<> +// ast_object_body is `{ ... }` inside object initialization, it contains fields +// examples: see below +struct Vertex final : ASTExprVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return child(i)->as(); } + std::vector get_all_fields() const { return children; } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTExprVararg(ast_object_body, loc, std::move(fields)) {} +}; + +template<> +// ast_object_literal is creating an instance of a struct with initial values of fields, like objects in TypeScript +// example: `Point { ... }` (object creation has type_node and body) +// example: `var v: Point = { ... }` (object creation has only body, struct_ref is determined from the left) +// example: `Wrapper { ... }` (also type_node and body, this type_node is resolved as instantiated generic struct) +struct Vertex final : ASTExprUnary { + StructPtr struct_ref = nullptr; // assigned at type inferring + AnyTypeV type_node; // not null for `T { ... }`, nullptr for plain `{ ... }` + + auto get_body() const { return child->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, AnyTypeV type_node, V body) + : ASTExprUnary(ast_object_literal, loc, body) + , type_node(type_node) {} }; @@ -732,10 +1017,10 @@ struct Vertex final : ASTStatementVararg { }; template<> -// ast_sequence is "some sequence of statements" -// example: function body is a sequence -// example: do while body is a sequence -struct Vertex final : ASTStatementVararg { +// ast_block_statement is "{ statement; statement }" (trailing semicolon is optional) +// example: function body is a block +// example: do while body is a block +struct Vertex final : ASTStatementVararg { SrcLocation loc_end; AnyV first_unreachable = nullptr; @@ -744,9 +1029,10 @@ struct Vertex final : ASTStatementVararg { Vertex* mutate() const { return const_cast(this); } void assign_first_unreachable(AnyV first_unreachable); + void assign_new_children(std::vector&& children); - Vertex(SrcLocation loc, SrcLocation loc_end, std::vector items) - : ASTStatementVararg(ast_sequence, loc, std::move(items)) + Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) + : ASTStatementVararg(ast_block_statement, loc, std::move(items)) , loc_end(loc_end) {} }; @@ -756,7 +1042,7 @@ template<> // note, that for `return;` (without a value, meaning "void"), in AST, it's stored as empty expression struct Vertex : ASTStatementUnary { AnyExprV get_return_value() const { return child_as_expr(); } - bool has_return_value() const { return child->type != ast_empty_expression; } + bool has_return_value() const { return child->kind != ast_empty_expression; } Vertex(SrcLocation loc, AnyExprV child) : ASTStatementUnary(ast_return_statement, loc, child) {} @@ -771,10 +1057,10 @@ struct Vertex final : ASTStatementVararg { bool is_ifnot; // if(!cond), to generate more optimal fift code AnyExprV get_cond() const { return child_as_expr(0); } - auto get_if_body() const { return children.at(1)->as(); } - auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + auto get_if_body() const { return children.at(1)->as(); } + auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) - Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) , is_ifnot(is_ifnot) {} }; @@ -784,9 +1070,9 @@ template<> // example: `repeat (10) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} }; @@ -795,9 +1081,9 @@ template<> // example: `while (x > 0) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} }; @@ -805,10 +1091,10 @@ template<> // ast_do_while_statement is a standard "do while" loop // example: `do { ... } while (x > 0);` struct Vertex final : ASTStatementVararg { - auto get_body() const { return children.at(0)->as(); } + auto get_body() const { return children.at(0)->as(); } AnyExprV get_cond() const { return child_as_expr(1); } - Vertex(SrcLocation loc, V body, AnyExprV cond) + Vertex(SrcLocation loc, V body, AnyExprV cond) : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} }; @@ -818,7 +1104,7 @@ template<> // when thrown arg is missing, it's stored as empty expression struct Vertex final : ASTStatementVararg { AnyExprV get_thrown_code() const { return child_as_expr(0); } - bool has_thrown_arg() const { return child_as_expr(1)->type != ast_empty_expression; } + bool has_thrown_arg() const { return child_as_expr(1)->kind != ast_empty_expression; } AnyExprV get_thrown_arg() const { return child_as_expr(1); } Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) @@ -842,11 +1128,11 @@ template<> // there are two formal "arguments" of catch: excNo and arg, but both can be omitted // when omitted, they are stored as underscores, so len of a catch tensor is always 2 struct Vertex final : ASTStatementVararg { - auto get_try_body() const { return children.at(0)->as(); } + auto get_try_body() const { return children.at(0)->as(); } auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 - auto get_catch_body() const { return children.at(2)->as(); } + auto get_catch_body() const { return children.at(2)->as(); } - Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) + Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} }; @@ -876,12 +1162,14 @@ struct Vertex final : ASTStatementVararg { template<> // ast_genericsT_item is generics T at declaration // example: `fun f` has a list of 2 generic Ts +// example: `struct Params` has 1 generic T with default struct Vertex final : ASTOtherLeaf { std::string_view nameT; + AnyTypeV default_type_node; // exists for ``, nullptr otherwise - Vertex(SrcLocation loc, std::string_view nameT) + Vertex(SrcLocation loc, std::string_view nameT, AnyTypeV default_type_node) : ASTOtherLeaf(ast_genericsT_item, loc) - , nameT(nameT) {} + , nameT(nameT), default_type_node(default_type_node) {} }; template<> @@ -902,14 +1190,11 @@ template<> // ast_instantiationT_item is manual substitution of generic T used in code, mostly for func calls // examples: `g()` / `t.tupleFirst()` / `f<(int, slice), builder>()` struct Vertex final : ASTOtherLeaf { - TypePtr substituted_type; - - Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr substituted_type); + AnyTypeV type_node; - Vertex(SrcLocation loc, TypePtr substituted_type) + Vertex(SrcLocation loc, AnyTypeV type_node) : ASTOtherLeaf(ast_instantiationT_item, loc) - , substituted_type(substituted_type) {} + , type_node(type_node) {} }; template<> @@ -919,28 +1204,25 @@ struct Vertex final : ASTOtherVararg { std::vector get_items() const { return children; } auto get_item(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector instantiationTs) + Vertex(SrcLocation loc, std::vector&& instantiationTs) : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} }; template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters +// example: `fun f(a: int = 0)` has 1 parameter with default value struct Vertex final : ASTOtherLeaf { - LocalVarPtr param_ref = nullptr; // filled on resolve identifiers std::string_view param_name; - TypePtr declared_type; + AnyTypeV type_node; // always exists, typing parameters is mandatory + AnyExprV default_value; // default value of the parameter or nullptr bool declared_as_mutate; // declared as `mutate param_name` bool is_underscore() const { return param_name.empty(); } - Vertex* mutate() const { return const_cast(this); } - void assign_param_ref(LocalVarPtr param_ref); - void assign_resolved_type(TypePtr declared_type); - - Vertex(SrcLocation loc, std::string_view param_name, TypePtr declared_type, bool declared_as_mutate) + Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, AnyExprV default_value, bool declared_as_mutate) : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), declared_type(declared_type), declared_as_mutate(declared_as_mutate) {} + , param_name(param_name), type_node(type_node), default_value(default_value), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -962,15 +1244,16 @@ template<> // ast_annotation is @annotation above a declaration // example: `@pure fun ...` struct Vertex final : ASTOtherVararg { + std::string_view name; AnnotationKind kind; auto get_arg() const { return children.at(0)->as(); } static AnnotationKind parse_kind(std::string_view name); - Vertex(SrcLocation loc, AnnotationKind kind, V arg_probably_empty) + Vertex(SrcLocation loc, std::string_view name, AnnotationKind kind, V arg_probably_empty) : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) - , kind(kind) {} + , name(name), kind(kind) {} }; template<> @@ -981,28 +1264,29 @@ template<> // their body is either sequence (regular code function), or `asm`, or `builtin` struct Vertex final : ASTOtherVararg { auto get_identifier() const { return children.at(0)->as(); } - int get_num_params() const { return children.at(1)->as()->size(); } + int get_num_params() const { return children.at(1)->as()->size(); } auto get_param_list() const { return children.at(1)->as(); } auto get_param(int i) const { return children.at(1)->as()->get_param(i); } - AnyV get_body() const { return children.at(2); } // ast_sequence / ast_asm_body + AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register - TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" + AnyTypeV receiver_type_node; // for `fun builder.storeInt`, here is `builder` + AnyTypeV return_type_node; // if unspecified (nullptr), means "auto infer" V genericsT_list; // for non-generics it's nullptr - td::RefInt256 method_id; // specified via @method_id annotation + td::RefInt256 tvm_method_id; // specified via @method_id annotation int flags; // from enum in FunctionData + FunctionInlineMode inline_mode; // from annotations like `@inline` or auto-detected "in-place" - bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } - bool is_code_function() const { return children.at(2)->type == ast_sequence; } - bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } + bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } + bool is_code_function() const { return children.at(2)->kind == ast_block_statement; } + bool is_builtin_function() const { return children.at(2)->kind == ast_empty_statement; } Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - void assign_resolved_type(TypePtr declared_return_type); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, TypePtr declared_return_type, V genericsT_list, td::RefInt256 method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags, FunctionInlineMode inline_mode) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , declared_return_type(declared_return_type), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} + , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags), inline_mode(inline_mode) {} }; template<> @@ -1010,18 +1294,17 @@ template<> // example: `global g: int;` // note, that globals don't have default values, since there is no single "entrypoint" for a contract struct Vertex final : ASTOtherVararg { - GlobalVarPtr var_ref = nullptr; // filled after register - TypePtr declared_type; // filled always, typing globals is mandatory + GlobalVarPtr glob_ref = nullptr; // filled after register + AnyTypeV type_node; // always exists, typing globals is mandatory auto get_identifier() const { return children.at(0)->as(); } Vertex* mutate() const { return const_cast(this); } - void assign_var_ref(GlobalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); + void assign_glob_ref(GlobalVarPtr glob_ref); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node) : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> @@ -1029,18 +1312,85 @@ template<> // example: `const op = 0x123;` struct Vertex final : ASTOtherVararg { GlobalConstPtr const_ref = nullptr; // filled after register - TypePtr declared_type; // not null for `const op: int = ...` + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr auto get_identifier() const { return children.at(0)->as(); } AnyExprV get_init_value() const { return child_as_expr(1); } Vertex* mutate() const { return const_cast(this); } void assign_const_ref(GlobalConstPtr const_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type, AnyExprV init_value) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node, AnyExprV init_value) : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) - , declared_type(declared_type) {} + , type_node(type_node) {} +}; + +template<> +// ast_type_alias_declaration is declaring a structural type alias (fully interchangeable with original type) +// example: `type UserId = int;` +// see TypeDataAlias in type-system.h +struct Vertex final : ASTOtherVararg { + AliasDefPtr alias_ref = nullptr; // filled after register + V genericsT_list; // exists for `type Response`; otherwise, nullptr + AnyTypeV underlying_type_node; // at the right of `=` + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_alias_ref(AliasDefPtr alias_ref); + + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, AnyTypeV underlying_type_node) + : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) + , genericsT_list(genericsT_list), underlying_type_node(underlying_type_node) {} +}; + +template<> +// ast_struct_field is one field at struct declaration +// example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields +struct Vertex final : ASTOtherVararg { + AnyTypeV type_node; // always exists, typing struct fields is mandatory + AnyExprV default_value; // nullptr if no default + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) + : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + , type_node(type_node), default_value(default_value) {} +}; + +template<> +// ast_struct_body is `{ ... }` inside struct declaration, it contains fields +// example: `struct Storage { owner: User; validUntil: int }` its body contains 2 fields +struct Vertex final : ASTOtherVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return children.at(i)->as(); } + const std::vector& get_all_fields() const { return children; } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} +}; + +template<> +// ast_struct_declaration is declaring a struct with fields (each having declared_type), like interfaces in TypeScript +// example: `struct Storage { owner: User; validUntil: int }` +// example: `struct(0x0012) CounterIncrement { byValue: int32; }` (0x0012 is opcode, len 16) +// currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body +struct Vertex final : ASTOtherVararg { + StructPtr struct_ref = nullptr; // filled after register + V genericsT_list; // exists for `Wrapper`; otherwise, nullptr + StructData::Overflow1023Policy overflow1023_policy; + + auto get_identifier() const { return children.at(0)->as(); } + bool has_opcode() const { return children.at(1)->kind != ast_empty_expression; } + auto get_opcode() const { return children.at(1)->as(); } + auto get_struct_body() const { return children.at(2)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, StructData::Overflow1023Policy overflow1023_policy, AnyExprV opcode, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, opcode, struct_body}) + , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index cb89c984c..dc2795c89 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -18,6 +18,7 @@ #include "compiler-state.h" #include "type-system.h" #include "generics-helpers.h" +#include "ast.h" namespace tolk { using namespace std::literals::string_literals; @@ -32,7 +33,7 @@ static std::vector define_builtin_parameters(const std::vector(params_types.size()); ++i) { - LocalVarData p_sym("", {}, params_types[i], (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); + LocalVarData p_sym("", {}, params_types[i], nullptr, (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); parameters.push_back(std::move(p_sym)); } @@ -40,30 +41,31 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); G.symtable.add_function(f_sym); } -static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const AsmOp& macro, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(make_simple_compile(macro)), nullptr); - G.symtable.add_function(f_sym); -} - -static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, - std::initializer_list arg_order, std::initializer_list ret_order) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); +static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, + std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { + std::string method_name = name.substr(name.find('.') + 1); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); + G.all_methods.push_back(f_sym); } void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, - SrcLocation where) const { - dest.append(simple_compile(out, in, where)); + SrcLocation loc) const { + dest << simple_compile(out, in, loc); } -void FunctionBodyAsm::compile(AsmOpList& dest) const { - dest.append(ops); +void FunctionBodyAsm::compile(AsmOpList& dest, SrcLocation loc) const { + for (const AsmOp& op : ops) { + AsmOp copy = op; + copy.loc = loc; + dest << std::move(copy); + } } @@ -331,101 +333,89 @@ bool VarDescr::always_neq(const VarDescr& other) const { (always_odd() && other.always_even()); } -AsmOp exec_op(std::string op) { - return AsmOp::Custom(op); +AsmOp exec_op(SrcLocation loc, std::string op) { + return AsmOp::Custom(loc, op); } -AsmOp exec_op(std::string op, int args, int retv = 1) { - return AsmOp::Custom(op, args, retv); +AsmOp exec_op(SrcLocation loc, std::string op, int args, int retv = 1) { + return AsmOp::Custom(loc, op, args, retv); } -AsmOp exec_arg_op(std::string op, long long arg) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str()); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg_op(std::string op, td::RefInt256 arg) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str()); -} - -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); -} - -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv) { +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv) { std::ostringstream os; os << imm1 << ' ' << imm2 << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp push_const(td::RefInt256 x) { - return AsmOp::IntConst(std::move(x)); +AsmOp push_const(SrcLocation loc, td::RefInt256 x) { + return AsmOp::IntConst(loc, std::move(x)); } -AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_add(x.val, y.val); if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { y.unused(); if (y.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*y.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", y.int_const, 1); } if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { x.unused(); if (x.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*x.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", x.int_const, 1); + return exec_arg_op(loc, "ADDCONST", x.int_const, 1); } - return exec_op("ADD", 2); + return exec_op(loc, "ADD", 2); } -AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_sub(x.val, y.val); if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { @@ -434,121 +424,121 @@ AsmOp compile_sub(std::vector& res, std::vector& args, SrcLo return {}; } if (*y.int_const == 1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } if (*y.int_const == -1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } - return exec_arg_op("ADDCONST", -y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", -y.int_const, 1); } if (x.always_zero()) { x.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_op("SUB", 2); + return exec_op(loc, "SUB", 2); } -AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_negate(x.val); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } -AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = x.val; - return AsmOp::Nop(); + return AsmOp::Nop(loc); } -AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation where, bool for_int_arg) { +static AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation loc, bool for_int_arg) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const == 0 ? -1 : 0); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = VarDescr::ValBool; // for integers, `!var` is `var != 0` // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) - return for_int_arg ? exec_op("0 EQINT", 1) : exec_op("NOT", 1); + return for_int_arg ? exec_op(loc, "0 EQINT", 1) : exec_op(loc, "NOT", 1); } -AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_and(x.val, y.val); - return exec_op("AND", 2); + return exec_op(loc, "AND", 2); } -AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_or(x.val, y.val); - return exec_op("OR", 2); + return exec_op(loc, "OR", 2); } -AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_xor(x.val, y.val); - return exec_op("XOR", 2); + return exec_op(loc, "XOR", 2); } -AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_not(x.val); - return exec_op("NOT", 1); + return exec_op(loc, "NOT", 1); } -AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where) { +static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mul(x.val, y.val); if (y.is_int_const()) { @@ -559,23 +549,23 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*y.int_const == 1 && x.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", y.int_const, 1); + return exec_arg_op(loc, "MULCONST", y.int_const, 1); } if (k > 0) { y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const()) { @@ -586,48 +576,48 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*x.int_const == 1 && y.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", x.int_const, 1); + return exec_arg_op(loc, "MULCONST", x.int_const, 1); } if (k > 0) { x.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } - return exec_op("MUL", 2); + return exec_op(loc, "MUL", 2); } -AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_mul_internal(res[0], args[0], args[1], where); + return compile_mul_internal(res[0], args[0], args[1], loc); } -AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "lshift argument is out of range"); + throw ParseError(loc, "lshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_lshift(x.val, y.val); @@ -636,38 +626,38 @@ AsmOp compile_lshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (x.is_int_const()) { auto xv = x.int_const->to_long(); if (xv == 1) { x.unused(); - return exec_op("POW2", 1); + return exec_op(loc, "POW2", 1); } if (xv == -1) { x.unused(); - return exec_op("-1 PUSHINT SWAP LSHIFT", 1); + return exec_op(loc, "-1 PUSHINT SWAP LSHIFT", 1); } } - return exec_op("LSHIFT", 2); + return exec_op(loc, "LSHIFT", 2); } -AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "rshift argument is out of range"); + throw ParseError(loc, "rshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_rshift(x.val, y.val); @@ -677,36 +667,36 @@ AsmOp compile_rshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op(rshift + "#", k, 1); + return exec_arg_op(loc, rshift + "#", k, 1); } - return exec_op(rshift, 2); + return exec_op(loc, rshift, 2); } -AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where, int round_mode) { +static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc, int round_mode) { if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { y.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -715,44 +705,44 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "DIV"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation where, int round_mode) { +static AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_div_internal(res[0], args[0], args[1], where, round_mode); + return compile_div_internal(res[0], args[0], args[1], loc, round_mode); } -AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); y.unused(); r.set_const(td::zero_refint()); - return push_const(r.int_const); + return push_const(loc, r.int_const); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -761,29 +751,29 @@ AsmOp compile_mod(std::vector& res, std::vector& args, SrcLo if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "MOD"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *z.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); z.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (x.always_zero() || y.always_zero()) { // dubious optimization for z=0... @@ -791,26 +781,26 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr y.unused(); z.unused(); r.set_const(td::make_refint(0)); - return push_const(r.int_const); + return push_const(loc, r.int_const); } char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*z.int_const == 1) { z.unused(); - return compile_mul_internal(r, x, y, where); + return compile_mul_internal(r, x, y, loc); } } if (y.is_int_const() && *y.int_const == 1) { y.unused(); - return compile_div_internal(r, x, z, where, round_mode); + return compile_div_internal(r, x, z, loc, round_mode); } if (x.is_int_const() && *x.int_const == 1) { x.unused(); - return compile_div_internal(r, y, z, where, round_mode); + return compile_div_internal(r, y, z, loc, round_mode); } if (z.is_int_const()) { int k = is_pos_pow2(z.int_const); @@ -820,7 +810,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op + '#', k, 2); + return exec_arg_op(loc, op + '#', k, 2); } } if (y.is_int_const()) { @@ -831,7 +821,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } if (x.is_int_const()) { @@ -842,17 +832,22 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } std::string op = "MULDIV"; if (c) { op += c; } - return exec_op(op, 3); + return exec_op(loc, op, 3); +} + +// fun mulDivMod(x: int, y: int, z: int): (int, int) asm "MULDIVMOD"; +static AsmOp compile_muldivmod(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "MULDIVMOD", 3, 2); } -int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { +static int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { int s = td::cmp(x, y); if (mode == 7) { return s; @@ -866,7 +861,7 @@ int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { // 2 -> constant 0 // 1 -> constant -1 // 3 -> 0 or -1 -int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { +static int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { switch (mode) { case 1: // > return x.always_greater(y) ? 1 : (x.always_leq(y) ? 2 : 3); @@ -893,7 +888,7 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } } -AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { +static AsmOp compile_cmp_int(std::vector& res, std::vector& args, SrcLocation loc, int mode) { tolk_assert(mode >= 1 && mode <= 7); tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; @@ -902,7 +897,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v != 0); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v != 0); } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; @@ -911,7 +906,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v - (v >> 2) - 2); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v & 1); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v & 1); } r.val = ~0; if (v & 1) { @@ -930,29 +925,31 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i if (mode != 7) { if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { y.unused(); - return exec_arg_op(cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); } if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { x.unused(); mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); - return exec_arg_op(cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); } } - return exec_op(cmp_names[mode], 2); + return exec_op(loc, cmp_names[mode], 2); } -AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; - if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + if (x.is_int_const() && x.int_const >= 0) { + // in Fift assembler, "N THROW" is valid if N < 2048; for big N (particularly, widely used 0xFFFF) + // we now still generate "N THROW", and later, in optimizer, transform it to "PUSHINT" + "THROWANY" x.unused(); - return exec_arg_op("THROW", x.int_const, 0, 0); + return exec_arg_op(loc, "THROW", x.int_const, 0, 0); } else { - return exec_op("THROWANY", 1, 0); + return exec_op(loc, "THROWANY", 1, 0); } } -AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 3); VarDescr &x = args[0], &y = args[1], &z = args[2]; if (!z.always_true() && !z.always_false()) { @@ -967,40 +964,75 @@ AsmOp compile_throw_if_unless(std::vector& res, std::vector& skip_cond = true; if (y.always_true() != mode) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return skip_cond ? exec_arg_op("THROW", x.int_const, 0, 0) : exec_arg_op("THROW"s + suff, x.int_const, 1, 0); + return skip_cond ? exec_arg_op(loc, "THROW", x.int_const, 0, 0) : exec_arg_op(loc, "THROW"s + suff, x.int_const, 1, 0); } else { - return skip_cond ? exec_op("THROWANY", 1, 0) : exec_op("THROWANY"s + suff, 2, 0); + return skip_cond ? exec_op(loc, "THROWANY", 1, 0) : exec_op(loc, "THROWANY"s + suff, 2, 0); } } -AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_calc_InMessage_originalForwardFee(std::vector& res, std::vector& args, SrcLocation loc) { + return exec_op(loc, "GETORIGINALFWDFEE", 2); +} + +static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, std::vector& args, SrcLocation loc) { + // instead of "0 INMSGPARAM", generate "INMSG_BOUNCE", etc. — these are aliases in Asm.fif + static const char* aliases[] = { + "INMSG_BOUNCE", "INMSG_BOUNCED", "INMSG_SRC", "INMSG_FWDFEE", "INMSG_LT", "INMSG_UTIME", "INMSG_ORIGVALUE", "INMSG_VALUE", "INMSG_VALUEEXTRA", "INMSG_STATEINIT", + }; + tolk_assert(res.size() == 1 && args.size() == 1 && args[0].is_int_const()); + args[0].unused(); + size_t idx = args[0].int_const->to_long(); + if (idx < std::size(aliases)) { + return exec_op(loc, aliases[idx], 0); + } + return exec_arg_op(loc, "INMSGPARAM", args[0].int_const, 1); +} + +static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op("THROWARG", x.int_const, 1, 0); + return exec_arg_op(loc, "THROWARG", x.int_const, 1, 0); } else { - return exec_op("THROWARGANY", 2, 0); + return exec_op(loc, "THROWARGANY", 2, 0); } } -AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { +// `x ? y : z` can be compiled as `CONDSEL` asm instruction if y and z are don't require evaluation +static AsmOp compile_ternary_as_condsel(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 3); + VarDescr& cond = args[0]; // args = [ cond, when_true, when_false ] + if (cond.always_true()) { + cond.unused(); + args[2].unused(); + return AsmOp::Nop(loc); + } + if (cond.always_false()) { + cond.unused(); + args[1].unused(); + return AsmOp::Nop(loc); + } + return exec_op(loc, "CONDSEL", 3, 1); +} + +static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); - return AsmOp::Const(val ? "TRUE" : "FALSE"); + return AsmOp::Const(loc, val ? "TRUE" : "FALSE"); } -// fun loadInt (mutate s: slice, len: int): int asm(s len -> 1 0) "LDIX"; -// fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; -// fun preloadInt (s: slice, len: int): int asm "PLDIX"; -// fun preloadUint(s: slice, len: int): int asm "PLDUX"; -AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { +// fun slice.loadInt (mutate self, len: int): int asm(s len -> 1 0) "LDIX"; +// fun slice.loadUint (mutate self, len: int): int asm( -> 1 0) "LDUX"; +// fun slice.preloadInt (self, len: int): int asm "PLDIX"; +// fun slice.preloadUint(self, len: int): int asm "PLDUX"; +static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); @@ -1015,27 +1047,110 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, } if (v > 0) { y.unused(); - return exec_arg_op((fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); } } - return exec_op((fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); + return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); +} + +// fun slice.__loadVarInt(mutate self, bits: int, unsigned: bool): int +static AsmOp compile_fetch_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[1].is_int_const() && args[2].is_int_const()); + long n_bits = args[1].int_const->to_long(); + long is_unsigned = args[2].int_const->to_long(); + + args[1].unused(); + args[2].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "LDVARUINT16" : "LDVARINT16", 1, 2); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "LDVARUINT32" : "LDVARINT32", 1, 2); + } + tolk_assert(false); } -// fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; -// fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; -AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { +// fun builder.storeInt (mutate self, x: int, len: int): self asm(x b len) "STIX"; +// fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; +static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); + auto& x = args[1]; auto& z = args[2]; + // purpose: to merge consecutive `b.storeUint(0, 1).storeUint(1, 1)` into one "1 PUSHINT + 2 STU", + // when constant arguments are passed, keep them as a separate (fake) instruction, to be handled by optimizer later + bool value_and_len_is_const = z.is_int_const() && x.is_int_const(); + if (value_and_len_is_const && G.settings.optimization_level >= 2) { + // don't handle negative numbers or potential overflow, merging them is incorrect + bool value_is_safe = sgnd + ? x.int_const >= 0 && z.int_const < 64 && x.int_const < (1ULL << (z.int_const->to_long() - 1)) + : x.int_const >= 0; + if (value_is_safe && z.int_const > 0 && z.int_const <= (255 + !sgnd)) { + z.unused(); + x.unused(); + return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); + } + } if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op("ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); } - return exec_op("ST"s + (sgnd ? "IX" : "UX"), 3, 1); + return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); +} + +// fun builder.__storeVarInt (mutate self, x: int, bits: int, unsigned: bool): self +static AsmOp compile_store_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 4 && res.size() == 1); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[2].is_int_const() && args[3].is_int_const()); + long n_bits = args[2].int_const->to_long(); + long is_unsigned = args[3].int_const->to_long(); + + args[2].unused(); + args[3].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "STVARUINT16" : "STVARINT16", 2, 1); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "STVARUINT32" : "STVARINT32", 2, 1); + } + tolk_assert(false); +} + +// fun builder.storeBool(mutate self, value: bool): self asm( -> 1 0) "1 STI"; +static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeBool(const_bool)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 1", 1); + } + if (v.is_int_const() && v.int_const == -1 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 1 1", 1); + } + return exec_op(loc, "1 STI", 2, 1); +} + +// fun builder.storeCoins(mutate self, value: coins): self asm "STGRAMS"; +static AsmOp compile_store_coins(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeCoins(const_zero)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 4", 1); + } + return exec_op(loc, "STGRAMS", 2, 1); } -// fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" -// fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" -AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { +// fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" +// fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" +static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; @@ -1043,39 +1158,138 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg v = (int)y.int_const->to_long(); if (v > 0) { y.unused(); - return exec_arg_op(fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); } } - return exec_op(fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); + return exec_op(loc, fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); } -// fun tupleAt(t: tuple, index: int): X asm "INDEXVAR"; -AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation) { +// fun slice.tryStripPrefix(mutate self, prefix: int, prefixLen: int): bool +// constructs "x{...} SDBEGINSQ" for constant arguments +AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + auto& prefix = args[1]; + auto& prefix_len = args[2]; + if (prefix.is_int_const() && prefix.int_const >= 0 && prefix.int_const->signed_fits_bits(50) && + prefix_len.is_int_const() && prefix_len.int_const > 0 && prefix_len.int_const->signed_fits_bits(16)) { + prefix.unused(); + prefix_len.unused(); + StructData::PackOpcode opcode(prefix.int_const->to_long(), static_cast(prefix_len.int_const->to_long())); + return AsmOp::Custom(loc, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); + } + throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); +} + +// fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" +AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& len = args[1]; + // same technique as for storeUint: + // consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 + // to track this, represent it as a separate fake instruction to be detected by optimizer later + if (len.is_int_const() && len.int_const >= 0 && G.settings.optimization_level >= 2) { + len.unused(); + return AsmOp::Custom(loc, "MY_skip_bits " + len.int_const->to_dec_string(), 1); + } + return exec_op(loc, "SDSKIPFIRST", 2, 1); +} + + +// fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; +static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("INDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "INDEX", y.int_const, 1, 1); } - return exec_op("INDEXVAR", 2, 1); + return exec_op(loc, "INDEXVAR", 2, 1); } -// fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; -AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation) { +// fun tuple.set(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; +static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("SETINDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "SETINDEX", y.int_const, 1, 1); + } + return exec_op(loc, "SETINDEXVAR", 2, 1); +} + +// fun debug.dumpStack(): void asm "DUMPSTK"; +static AsmOp compile_dumpstk(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "DUMPSTK", 0, 0); +} + +// fun debug.printString(x: T): void asm "STRDUMP"; +static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); +} + +// fun debug.print(x: T): void; +static AsmOp compile_debug_print_to_string(std::vector&, std::vector& args, SrcLocation loc) { + int n = static_cast(args.size()); + if (n == 1) { // most common case + return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); + } + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + std::string cmd; + for (int i = n - 1; i >= 0; --i) { + cmd += "s" + std::to_string(i) + " DUMP "; + } + cmd += std::to_string(n); + cmd += " BLKDROP"; + return AsmOp::Custom(loc, cmd, n, n); +} + +// fun T.__toTuple(self): void; (T can be any number of slots, it works for structs and tensors) +static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); +} + +// fun sizeof(anything: T): int; // (returns the number of stack elements) +static AsmOp compile_any_object_sizeof(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + res[0].set_const(n); + for (int i = 0; i < n; ++i) { + args[i].unused(); } - return exec_op("SETINDEXVAR", 2, 1); + return AsmOp::IntConst(loc, td::make_refint(n)); +} + +// fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time +// same for stringCrc32(constString: slice) and others +AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { + // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime + tolk_assert(false); + return AsmOp::Nop(loc); +} + +// `null` literal is under the hood transformed to PUSHNULL +static AsmOp compile_push_null(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Const(loc, "PUSHNULL"); } // fun __isNull(X arg): bool -AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 1 && res.size() == 1); res[0].val = VarDescr::ValBool; - return exec_op("ISNULL", 1, 1); + return exec_op(loc, "ISNULL", 1, 1); +} + +// fun __expect_type(, ""): void; +static AsmOp compile_expect_type(std::vector&, std::vector&, SrcLocation loc) { + // handled by type checker, does nothing at runtime + return AsmOp::Nop(loc); } @@ -1087,19 +1301,32 @@ void define_builtins() { TypePtr Bool = TypeDataBool::create(); TypePtr Slice = TypeDataSlice::create(); TypePtr Builder = TypeDataBuilder::create(); + TypePtr Address = TypeDataAddress::create(); TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); - std::vector itemsT; - itemsT.emplace_back("T"); TypePtr typeT = TypeDataGenericT::create("T"); - const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::move(itemsT)); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 0); + const GenericsDeclaration* declReceiverT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 1); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; std::vector ParamsInt3 = {Int, Int, Int}; std::vector ParamsSliceInt = {Slice, Int}; + // these types are defined in stdlib, currently unknown + // see patch_builtins_after_stdlib_loaded() below + TypePtr debug = TypeDataUnknown::create(); + TypePtr CellT = TypeDataUnknown::create(); + TypePtr PackOptions = TypeDataUnknown::create(); + TypePtr UnpackOptions = TypeDataUnknown::create(); + TypePtr CreateMessageOptions = TypeDataUnknown::create(); + TypePtr CreateExternalLogMessageOptions = TypeDataUnknown::create(); + TypePtr OutMessage = TypeDataUnknown::create(); + TypePtr AddressShardingOptions = TypeDataUnknown::create(); + TypePtr AutoDeployAddress = TypeDataUnknown::create(); + const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); + // builtin operators // they are internally stored as functions, because at IR level, there is no difference // between calling `userAdd(a,b)` and `_+_(a,b)` @@ -1167,37 +1394,37 @@ void define_builtins() { compile_bitwise_xor, FunctionData::flagMarkedAsPure); define_builtin_func("_==_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 2), + std::bind(compile_cmp_int, _1, _2, _3, 2), FunctionData::flagMarkedAsPure); define_builtin_func("_!=_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 5), + std::bind(compile_cmp_int, _1, _2, _3, 5), FunctionData::flagMarkedAsPure); define_builtin_func("_<_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 4), + std::bind(compile_cmp_int, _1, _2, _3, 4), FunctionData::flagMarkedAsPure); define_builtin_func("_>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 1), + std::bind(compile_cmp_int, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("_<=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 6), + std::bind(compile_cmp_int, _1, _2, _3, 6), FunctionData::flagMarkedAsPure); define_builtin_func("_>=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 3), + std::bind(compile_cmp_int, _1, _2, _3, 3), FunctionData::flagMarkedAsPure); define_builtin_func("_<=>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 7), + std::bind(compile_cmp_int, _1, _2, _3, 7), FunctionData::flagMarkedAsPure); // special function used for internal compilation of some lexical constructs // for example, `throw 123;` is actually calling `__throw(123)` define_builtin_func("__true", {}, Bool, nullptr, /* AsmOp::Const("TRUE") */ - std::bind(compile_bool_const, _1, _2, true), + std::bind(compile_bool_const, _1, _2, _3, true), FunctionData::flagMarkedAsPure); define_builtin_func("__false", {}, Bool, nullptr, /* AsmOp::Const("FALSE") */ - std::bind(compile_bool_const, _1, _2, false), + std::bind(compile_bool_const, _1, _2, _3, false), FunctionData::flagMarkedAsPure); define_builtin_func("__null", {}, typeT, declGenericT, - AsmOp::Const("PUSHNULL"), + compile_push_null, FunctionData::flagMarkedAsPure); define_builtin_func("__isNull", {typeT}, Bool, declGenericT, compile_is_null, @@ -1211,6 +1438,50 @@ void define_builtins() { define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, compile_throw_if_unless, 0); + define_builtin_func("__InMessage.originalForwardFee", ParamsInt2, Int, nullptr, + compile_calc_InMessage_originalForwardFee, + 0); + define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr, + compile_calc_InMessage_getInMsgParam, + 0); + define_builtin_method("builder.__storeVarInt", Builder, {Builder, Int, Int, Bool}, Unit, nullptr, + compile_store_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); + define_builtin_method("slice.__loadVarInt", Slice, {Slice, Int, Bool}, Int, nullptr, + compile_fetch_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("__condsel", ParamsInt3, Int, nullptr, + compile_ternary_as_condsel, + 0); + + // compile-time only functions, evaluated essentially at compile-time, no runtime implementation + // they are placed in stdlib and marked as `builtin` + // note their parameter being `unknown`: in order to `ton(1)` pass type inferring but fire a more gentle error later + define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringCrc32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringCrc16", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringSha256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringSha256_32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); + define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) @@ -1224,59 +1495,207 @@ void define_builtins() { std::bind(compile_muldiv, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, - AsmOp::Custom("MULDIVMOD", 3, 2), + compile_muldivmod, FunctionData::flagMarkedAsPure); - define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, true), + define_builtin_method("slice.loadInt", Slice, ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, _3, true, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, false), + define_builtin_method("slice.loadUint", Slice, ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, _3, true, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, true), + define_builtin_method("slice.loadBits", Slice, ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, true), + define_builtin_method("slice.skipBits", Slice, ParamsSliceInt, Slice, nullptr, + compile_skip_bits_in_slice, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); + define_builtin_method("slice.preloadInt", Slice, ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, false), + define_builtin_method("slice.preloadUint", Slice, ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, _3, false, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, false), + define_builtin_method("slice.preloadBits", Slice, ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, true), + define_builtin_method("slice.tryStripPrefix", Slice, {Slice, Int, Int}, Bool, nullptr, + compile_slice_sdbeginsq, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); + define_builtin_method("builder.storeInt", Builder, {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, false), + define_builtin_method("builder.storeUint", Builder, {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, - compile_tuple_at, + define_builtin_method("builder.storeBool", Builder, {Builder, Bool}, Unit, nullptr, + compile_store_bool, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0}, {}); + define_builtin_method("builder.storeCoins", Builder, {Builder, TypeDataCoins::create()}, Unit, nullptr, + compile_store_coins, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); + define_builtin_method("tuple.get", Tuple, {Tuple, Int}, typeT, declGenericT, + compile_tuple_get, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("tupleSetAt", {Tuple, typeT, Int}, Unit, declGenericT, + define_builtin_method("tuple.set", Tuple, {Tuple, typeT, Int}, Unit, declGenericT, compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); - define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, - AsmOp::Custom("s0 DUMP DROP", 1, 1), - 0); - define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, - AsmOp::Custom("STRDUMP DROP", 1, 1), + define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, + compile_debug_print_to_string, + FunctionData::flagAllowAnyWidthT); + define_builtin_method("debug.printString", debug, {typeT}, Unit, declGenericT, + compile_strdump, 0); - define_builtin_func("debugDumpStack", {}, Unit, nullptr, - AsmOp::Custom("DUMPSTK", 0, 0), + define_builtin_method("debug.dumpStack", debug, {}, Unit, nullptr, + compile_dumpstk, 0); + define_builtin_func("sizeof", {typeT}, TypeDataInt::create(), declGenericT, + compile_any_object_sizeof, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); + + // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) + // they work with structs (or, more low-level, with arbitrary types) + define_builtin_method("T.toCell", typeT, {typeT, PackOptions}, CellT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromCell", typeT, {TypeDataCell::create(), UnpackOptions}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromSlice", typeT, {Slice, UnpackOptions}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.getDeclaredPackPrefix", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.getDeclaredPackPrefixLen", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.forceLoadLazyObject", typeT, {typeT}, Slice, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.loadAny", Slice, {Slice, UnpackOptions}, typeT, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.skipAny", Slice, {Slice, UnpackOptions}, Slice, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("builder.storeAny", Builder, {Builder, typeT, PackOptions}, Builder, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + + define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, - AsmOp::Nop(), + compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_inline", {Bool}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_lazy", {Slice}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); + define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, + compile_any_object_to_tuple, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); +} + +// there are some built-in functions that operate on types declared in stdlib (like Cell) +// that's why these symbols were undefined, and when builtins were registered, they were set to unknown +// after all files have been loaded, symbols have been registered, and aliases exist, +// we patch that earlier registered built-in functions providing types that now exist +void patch_builtins_after_stdlib_loaded() { + TypePtr typeT = TypeDataGenericT::create("T"); + StructPtr struct_debug = lookup_global_symbol("debug")->try_as(); + TypePtr debug = TypeDataStruct::create(struct_debug); + + lookup_function("debug.print")->mutate()->receiver_type = debug; + lookup_function("debug.printString")->mutate()->receiver_type = debug; + lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; + + StructPtr struct_ref_AddressShardingOptions = lookup_global_symbol("AddressShardingOptions")->try_as(); + StructPtr struct_ref_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + TypePtr AddressShardingOptions = TypeDataStruct::create(struct_ref_AddressShardingOptions); + TypePtr AutoDeployAddress = TypeDataStruct::create(struct_ref_AutoDeployAddress); + + lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->parameters[0].declared_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->parameters[0].declared_type = AutoDeployAddress; + + StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); + StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + StructPtr struct_ref_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); + TypePtr CellT = TypeDataGenericTypeWithTs::create(struct_ref_CellT, nullptr, {typeT}); + TypePtr PackOptions = TypeDataStruct::create(struct_ref_PackOptions); + TypePtr UnpackOptions = TypeDataStruct::create(struct_ref_UnpackOptions); + + // in stdlib, there is a default parameter `options = {}`; since default parameters are evaluated with AST, + // emulate its presence in built-in functions; it looks ugly, but currently I don't have a better solution + auto v_empty_PackOptions = createV({}, nullptr, createV({}, {})); + v_empty_PackOptions->assign_struct_ref(struct_ref_PackOptions); + v_empty_PackOptions->assign_inferred_type(PackOptions); + auto v_empty_UnpackOptions = createV({}, nullptr, createV({}, {})); + v_empty_UnpackOptions->assign_struct_ref(struct_ref_UnpackOptions); + v_empty_UnpackOptions->assign_inferred_type(UnpackOptions); + + lookup_function("T.toCell")->mutate()->declared_return_type = CellT; + lookup_function("T.toCell")->mutate()->parameters[1].declared_type = PackOptions; + lookup_function("T.toCell")->mutate()->parameters[1].default_value = v_empty_PackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("Cell.load")->mutate()->parameters[0].declared_type = CellT; + lookup_function("Cell.load")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("Cell.load")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("Cell.load")->mutate()->receiver_type = CellT; + lookup_function("slice.loadAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.loadAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].declared_type = PackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; + + StructPtr struct_ref_CreateMessageOptions = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr struct_ref_CreateExternalLogMessageOptions = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); + StructPtr struct_ref_OutMessage = lookup_global_symbol("OutMessage")->try_as(); + TypePtr CreateMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr CreateExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr OutMessage = TypeDataStruct::create(struct_ref_OutMessage); + + lookup_function("createMessage")->mutate()->parameters[0].declared_type = CreateMessageOptions; + lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; + lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = CreateExternalLogMessageOptions; + lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; } } // namespace tolk diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index ac1cf6391..f5ef89de3 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -82,38 +82,41 @@ void Stack::forget_const() { } } -void Stack::issue_pop(int i) { +void Stack::issue_pop(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Pop(i); + o << AsmOp::Pop(loc, i); } at(i) = get(0); s.pop_back(); modified(); + opt_show(); } -void Stack::issue_push(int i) { +void Stack::issue_push(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Push(i); + o << AsmOp::Push(loc, i); } s.push_back(get(i)); modified(); + opt_show(); } -void Stack::issue_xchg(int i, int j) { +void Stack::issue_xchg(SrcLocation loc, int i, int j) { validate(i); validate(j); if (i != j && get(i) != get(j)) { if (output_enabled()) { - o << AsmOp::Xchg(i, j); + o << AsmOp::Xchg(loc, i, j); } std::swap(at(i), at(j)); modified(); + opt_show(); } } -int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { +int Stack::drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var) { int dropped = 0, changes; do { changes = 0; @@ -122,7 +125,7 @@ int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { var_idx_t idx = at(i).first; if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { // unneeded - issue_pop(i); + issue_pop(loc, i); changes = 1; break; } @@ -138,7 +141,7 @@ void Stack::show() { os << ' '; o.show_var_ext(os, i); } - o << AsmOp::Comment(os.str()); + o << AsmOp::Comment({}, os.str()); mode |= _Shown; } @@ -172,17 +175,17 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { } } -void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { +void Stack::do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); tolk_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { - issue_push(i); + issue_push(loc, i); tolk_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } -void Stack::enforce_state(const StackLayout& req_stack) { +void Stack::enforce_state(SrcLocation loc, const StackLayout& req_stack) { int k = (int)req_stack.size(); for (int i = 0; i < k; i++) { var_idx_t x = req_stack[i]; @@ -191,18 +194,18 @@ void Stack::enforce_state(const StackLayout& req_stack) { } while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { // current TOS entry is unused in req_stack, drop it - issue_pop(0); + issue_pop(loc, 0); } int j = find(x); if (j >= depth() - i) { - issue_push(j); + issue_push(loc, j); j = 0; } - issue_xchg(j, depth() - i - 1); + issue_xchg(loc, j, depth() - i - 1); tolk_assert(s[i].first == x); } while (depth() > k) { - issue_pop(0); + issue_pop(loc, 0); } tolk_assert(depth() == k); for (int i = 0; i < k; i++) { @@ -220,12 +223,12 @@ void Stack::merge_const(const Stack& req_stack) { } } -void Stack::merge_state(const Stack& req_stack) { - enforce_state(req_stack.vars()); +void Stack::merge_state(SrcLocation loc, const Stack& req_stack) { + enforce_state(loc, req_stack.vars()); merge_const(req_stack); } -void Stack::rearrange_top(const StackLayout& top, std::vector last) { +void Stack::rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last) { while (last.size() < top.size()) { last.push_back(false); } @@ -250,24 +253,24 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { int j = find_outside(x, ss, ss + i); if (last[i]) { // rearrange x to be at s(ss-1) - issue_xchg(--ss, j); + issue_xchg(loc, --ss, j); tolk_assert(get(ss).first == x); } else { // create a new copy of x - issue_push(j); - issue_xchg(0, ss); + issue_push(loc, j); + issue_xchg(loc, 0, ss); tolk_assert(get(ss).first == x); } } tolk_assert(!ss); } -void Stack::rearrange_top(var_idx_t top, bool last) { +void Stack::rearrange_top(SrcLocation loc, var_idx_t top, bool last) { int i = find(top); if (last) { - issue_xchg(0, i); + issue_xchg(loc, 0, i); } else { - issue_push(i); + issue_push(loc, i); } tolk_assert(get(0).first == top); } @@ -280,7 +283,7 @@ bool Op::generate_code_step(Stack& stack) { bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin_function() && f_sym->name == "__throw") || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); if (!will_now_immediate_throw) { - stack.drop_vars_except(var_info); + stack.drop_vars_except(loc, var_info); stack.opt_show(); } @@ -290,9 +293,9 @@ bool Op::generate_code_step(Stack& stack) { case _Import: return true; case _Return: { - stack.enforce_state(left); + stack.enforce_state(loc, left); if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { - stack.o << "RETALT"; + stack.o << AsmOp::Custom(loc, "RETALT"); stack.o.retalt_inserted_ = true; } stack.opt_show(); @@ -306,11 +309,11 @@ bool Op::generate_code_step(Stack& stack) { auto cidx = stack.o.register_const(int_const); int i = stack.find_const(cidx); if (i < 0) { - stack.o << push_const(int_const); + stack.o << push_const(loc, int_const); stack.push_new_const(left[0], cidx); } else { tolk_assert(stack.at(i).second == cidx); - stack.do_copy_var(left[0], stack[i]); + stack.do_copy_var(loc, left[0], stack[i]); } return true; } @@ -319,7 +322,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused()) { return true; } - stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.o << AsmOp::Const(loc, "x{" + str_const + "} PUSHSLICE"); stack.push_new_var(left[0]); return true; } @@ -335,10 +338,10 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); } for (auto i : left) { stack.push_new_var(i); @@ -350,7 +353,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused() || disabled()) { return true; } - stack.o << "CONT:<{"; + stack.o << AsmOp::Custom(loc, "CONT:<{"); stack.o.indent(); if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) @@ -368,15 +371,15 @@ bool Op::generate_code_step(Stack& stack) { args0.emplace_back(0); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, where); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.push_new_var(left.at(0)); return true; } @@ -408,7 +411,7 @@ bool Op::generate_code_step(Stack& stack) { if (is_last) { stack.assign_var(--i, x); } else { - stack.do_copy_var(--i, x); + stack.do_copy_var(loc, --i, x); } } i = 0; @@ -428,15 +431,15 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); if (cl == _Tuple) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); tolk_assert(left.size() == 1); } else { - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); tolk_assert(right.size() == 1); } stack.s.resize(k); @@ -450,8 +453,10 @@ bool Op::generate_code_step(Stack& stack) { if (disabled()) { return true; } - // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function) - const std::vector* arg_order = f_sym ? f_sym->get_arg_order() : nullptr; + // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function); + // if f has arg_order, when it's safe, the compiler evaluates arguments in that order in advance (for fewer stack manipulations); + // when it's unsafe, arguments are evaluated left-to-right, and we need to match asm arg_order here + const std::vector* arg_order = f_sym && !arg_order_already_equals_asm() ? f_sym->get_arg_order() : nullptr; const std::vector* ret_order = f_sym ? f_sym->get_ret_order() : nullptr; tolk_assert(!arg_order || arg_order->size() == right.size()); tolk_assert(!ret_order || ret_order->size() == left.size()); @@ -475,24 +480,21 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right1) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right1, std::move(last)); + stack.rearrange_top(loc, right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { - if (stack.s[k + i].first != right1[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right1[i]); } auto exec_callxargs = [&](int args, int ret) { if (args <= 15 && ret <= 15) { - stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + stack.o << exec_arg2_op(loc, "CALLXARGS", args, ret, args + 1, ret); } else { tolk_assert(args <= 254 && ret <= 254); - stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); - stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); - stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + stack.o << AsmOp::Const(loc, PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(loc, PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom(loc, "CALLXVARARGS", args + 3, ret); } }; if (cl == _CallInd) { @@ -504,20 +506,27 @@ bool Op::generate_code_step(Stack& stack) { res.emplace_back(i); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) } else { - std::get(f_sym->body)->compile(stack.o, res, args, where); // compile res := f (args) + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } + std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } } else { - if (f_sym->is_inline() || f_sym->is_inline_ref()) { - stack.o << AsmOp::Custom(f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { + stack.o << AsmOp::Custom(loc, f_sym->name + "() INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(f_sym->name + (" PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(loc, f_sym->name + ("() PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } } + stack.modified(); stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { int j = ret_order ? ret_order->at(i) : i; @@ -531,21 +540,19 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { - if (stack.s[k + i].first != right[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " SETGLOB", 1, 0); + stack.modified(); } stack.s.resize(k); return true; @@ -558,7 +565,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } var_idx_t x = left[0]; - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -568,19 +575,19 @@ bool Op::generate_code_step(Stack& stack) { Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); stack.mode &= ~Stack::_InlineFunc; - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); block_other->generate_code_all(stack); if (!block_other->noreturn()) { next->generate_code_all(stack); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return false; } if (block1->is_empty() || block0->is_empty()) { @@ -589,88 +596,88 @@ bool Op::generate_code_step(Stack& stack) { // if (left) block0; ... // if (!left) block1; ... if (block->noreturn()) { - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; block->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}, stack_target{stack}; stack_target.disable_output(); - stack_target.drop_vars_except(next->var_info); + stack_target.drop_vars_except(loc, next->var_info); stack_copy.mode &= ~Stack::_InlineFunc; block->generate_code_all(stack_copy); - stack_copy.drop_vars_except(var_info); + stack_copy.drop_vars_except(loc, var_info); stack_copy.opt_show(); if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); if (!is0) { stack.merge_const(stack_copy); } return true; } // stack_copy.drop_vars_except(next->var_info); - stack_copy.enforce_state(stack_target.vars()); + stack_copy.enforce_state(loc, stack_target.vars()); stack_copy.opt_show(); if (stack_copy.vars() == stack.vars()) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.merge_const(stack_copy); return true; } stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } if (block0->noreturn() || block1->noreturn()) { bool is0 = block0->noreturn(); Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); block_other->generate_code_all(stack); return !block_other->noreturn(); } - stack.o << "IF:<{"; + stack.o << AsmOp::Custom(loc, "IF:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; block0->generate_code_all(stack_copy); - stack_copy.drop_vars_except(next->var_info); + stack_copy.drop_vars_except(loc, next->var_info); stack_copy.opt_show(); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); stack.mode &= ~Stack::_InlineFunc; block1->generate_code_all(stack); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } case _Repeat: { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -679,7 +686,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "REPEAT:<{"; + stack.o << AsmOp::Custom(loc, "REPEAT:<{"); stack.o.indent(); stack.forget_const(); if (block0->noreturn()) { @@ -693,47 +700,47 @@ bool Op::generate_code_step(Stack& stack) { stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "REPEATEND"; + stack.o << AsmOp::Custom(loc, "REPEATEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } } case _Again: { - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); if (block0->noreturn()) { stack.o.retalt_ = true; } if (!next->is_empty() || inline_func) { - stack.o << "AGAIN:<{"; + stack.o << AsmOp::Custom(loc, "AGAIN:<{"); stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "AGAINEND"; + stack.o << AsmOp::Custom(loc, "AGAINEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -745,7 +752,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "UNTIL:<{"; + stack.o << AsmOp::Custom(loc, "UNTIL:<{"); stack.o.indent(); stack.forget_const(); auto layout1 = stack.vars(); @@ -753,20 +760,20 @@ bool Op::generate_code_step(Stack& stack) { stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.s.pop_back(); stack.modified(); return true; } else { - stack.o << "UNTILEND"; + stack.o << AsmOp::Custom(loc, "UNTILEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -774,36 +781,36 @@ bool Op::generate_code_step(Stack& stack) { case _While: { // while (block0 | left) block1; ...next var_idx_t x = left[0]; - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); StackLayout layout1 = stack.vars(); bool next_empty = false && next->is_empty(); if (block0->noreturn()) { stack.o.retalt_ = true; } - stack.o << "WHILE:<{"; + stack.o << AsmOp::Custom(loc, "WHILE:<{"); stack.o.indent(); stack.forget_const(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); + stack.rearrange_top(loc, x, !next->var_info[x] && !block1->var_info[x]); stack.opt_show(); stack.s.pop_back(); stack.modified(); stack.o.undent(); Stack stack_copy{stack}; - stack.o << (next_empty ? "}>DO:" : "}>DO<{"); + stack.o << AsmOp::Custom(loc, next_empty ? "}>DO:" : "}>DO<{"); if (!next_empty) { stack.o.indent(); } stack_copy.opt_show(); block1->generate_code_all(stack_copy); - stack_copy.enforce_state(std::move(layout1)); + stack_copy.enforce_state(loc, std::move(layout1)); stack_copy.opt_show(); if (!next_empty) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { return false; @@ -816,7 +823,7 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn() || block1->noreturn()) { stack.o.retalt_ = true; } - Stack catch_stack{stack.o}; + Stack catch_stack{stack.o, 0}; std::vector catch_vars; std::vector catch_last; for (const VarDescr& var : block1->var_info.list) { @@ -834,37 +841,28 @@ bool Op::generate_code_step(Stack& stack) { } catch_stack.push_new_var(left[0]); catch_stack.push_new_var(left[1]); - stack.rearrange_top(catch_vars, catch_last); + stack.rearrange_top(loc, catch_vars, catch_last); stack.opt_show(); - stack.o << "c1 PUSH"; - stack.o << "c3 PUSH"; - stack.o << "c4 PUSH"; - stack.o << "c5 PUSH"; - stack.o << "c7 PUSH"; - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block1->noreturn()) { catch_stack.mode |= Stack::_NeedRetAlt; } block1->generate_code_all(catch_stack); - catch_stack.drop_vars_except(next->var_info); + catch_stack.drop_vars_except(loc, next->var_info); catch_stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c7 SETCONT"; - stack.o << "c5 SETCONT"; - stack.o << "c4 SETCONT"; - stack.o << "c3 SETCONT"; - stack.o << "c1 SETCONT"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "0b10111010 SETCONTMANY"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; - stack.o << std::to_string(end - begin) + " PUSHINT"; - stack.o << "-1 PUSHINT"; - stack.o << "SETCONTVARARGS"; + stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); + stack.o << AsmOp::Custom(loc, "-1 PUSHINT"); + stack.o << AsmOp::Custom(loc, "SETCONTVARARGS"); } stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); stack.modified(); - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block0->noreturn()) { stack.mode |= Stack::_NeedRetAlt; @@ -873,20 +871,20 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn()) { stack.s = std::move(catch_stack.s); } else if (!block1->noreturn()) { - stack.merge_state(catch_stack); + stack.merge_state(loc, catch_stack); } stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c1 PUSH"; - stack.o << "COMPOSALT"; - stack.o << "SWAP"; - stack.o << "TRY"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "c1 PUSH"); + stack.o << AsmOp::Custom(loc, "COMPOSALT"); + stack.o << AsmOp::Custom(loc, "SWAP"); + stack.o << AsmOp::Custom(loc, "TRY"); return true; } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in generate_code()"}; + throw ParseError(loc, "unknown operation in generate_code()"); } } @@ -899,20 +897,16 @@ void Op::generate_code_all(Stack& stack) { } } -void CodeBlob::generate_code(AsmOpList& out, int mode) { - Stack stack{out, mode}; +void CodeBlob::generate_code(std::ostream& os, int mode, int indent) const { + AsmOpList out_list(indent, &vars); + Stack stack{out_list, mode}; tolk_assert(ops && ops->cl == Op::_Import); - auto args = (int)ops->left.size(); + int n_import_width = static_cast(ops->left.size()); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); -} - -void CodeBlob::generate_code(std::ostream& os, int mode, int indent) { - AsmOpList out_list(indent, &vars); - generate_code(out_list, mode); + stack.apply_wrappers(fun_ref->loc, require_callxargs && (mode & Stack::_InlineAny) ? n_import_width : -1); if (G.settings.optimization_level >= 2) { optimize_code(out_list); } diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index 95a7e6a59..f5bda29e6 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -70,4 +70,12 @@ const std::vector& get_all_not_builtin_functions() { return G.all_functions; } +const std::vector& get_all_declared_constants() { + return G.all_constants; +} + +const std::vector& get_all_declared_structs() { + return G.all_structs; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 1d166a3ab..6685e092a 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -52,10 +52,11 @@ struct CompilerSettings { int verbosity = 0; int optimization_level = 2; bool stack_layout_comments = true; + bool tolk_src_as_line_comments = true; std::string output_filename; std::string boc_output_filename; - std::string stdlib_folder; // a path to tolk-stdlib/; files imported via @stdlib/xxx are there + std::string stdlib_folder; // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback FsReadCallback read_callback; @@ -95,10 +96,12 @@ struct CompilerState { GlobalSymbolTable symtable; PersistentHeapAllocator persistent_mem; - std::vector all_functions; // all user-defined (not built-in) functions, with generic instantiations - std::vector all_get_methods; + std::vector all_functions; // all user-defined (not built-in) global-scope functions, with generic instantiations + std::vector all_methods; // all user-defined and built-in extension methods for arbitrary types (receivers) + std::vector all_contract_getters; std::vector all_global_vars; std::vector all_constants; + std::vector all_structs; AllRegisteredSrcFiles all_src_files; bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 4d11b9223..20c316fdb 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -17,6 +17,7 @@ #include "constant-evaluator.h" #include "ast.h" #include "tolk.h" +#include "type-system.h" #include "openssl/digest.hpp" #include "crypto/common/util.h" #include "td/utils/crypto.h" @@ -24,6 +25,14 @@ namespace tolk { +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_const_string_required(SrcLocation loc, std::string_view f_name, std::string_view example_arg) { + std::string name = static_cast(f_name); + std::string example = static_cast(example_arg); + throw ParseError(loc, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); +} + + // parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" // based on unpack_std_smc_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) @@ -44,10 +53,10 @@ static bool parse_friendly_address(const char packed[48], ton::WorkchainId& work // parse address like "0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8" // based on StdAddress::parse_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) -static bool parse_raw_address(const std::string& acc_string, int& workchain, ton::StdSmcAddress& addr) { +static bool parse_raw_address(std::string_view acc_string, int& workchain, ton::StdSmcAddress& addr) { size_t pos = acc_string.find(':'); if (pos != std::string::npos) { - td::Result r_wc = td::to_integer_safe(acc_string.substr(0, pos)); + td::Result r_wc = td::to_integer_safe(td::Slice(acc_string.data(), pos)); if (r_wc.is_error()) { return false; } @@ -65,9 +74,9 @@ static bool parse_raw_address(const std::string& acc_string, int& workchain, ton int x; if (c >= '0' && c <= '9') { x = c - '0'; - } else if (c >= 'a' && c <= 'z') { + } else if (c >= 'a' && c <= 'f') { x = c - 'a' + 10; - } else if (c >= 'A' && c <= 'Z') { + } else if (c >= 'A' && c <= 'F') { x = c - 'A' + 10; } else { return false; @@ -82,236 +91,260 @@ static bool parse_raw_address(const std::string& acc_string, int& workchain, ton return true; } +static void parse_any_std_address(std::string_view str, SrcLocation loc, unsigned char (*data)[3 + 8 + 256]) { + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + throw ParseError(loc, "invalid standard address"); + } + if (workchain < -128 || workchain >= 128) { + throw ParseError(loc, "anycast addresses not supported"); + } -static std::string parse_vertex_string_const_as_slice(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 0: { - return td::hex_encode(str); - } - case 's': { - unsigned char buff[128]; - long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); - if (bits < 0) { - v->error("invalid hex bitstring constant '" + str + "'"); - } - return str; - } - case 'a': { // MsgAddress - ton::WorkchainId workchain; - ton::StdSmcAddress addr; - bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || - (str.size() != 48 && parse_raw_address(str, workchain, addr)); - if (!correct) { - v->error("invalid standard address '" + str + "'"); + td::bitstring::bits_store_long_top(*data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(*data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(*data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); +} + +// internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 +static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { + bool is_negative = false; + size_t i = 0; + + // handle optional leading sign + if (str[0] == '-') { + is_negative = true; + i++; + } else if (str[0] == '+') { + i++; + } + + // parse "0.05" into integer part (before dot) and fractional (after) + int64_t integer_part = 0; + int64_t fractional_part = 0; + int integer_digits = 0; + int fractional_digits = 0; + bool seen_dot = false; + + for (; i < str.size(); ++i) { + char c = str[i]; + if (c == '.') { + if (seen_dot) { + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } - if (workchain < -128 || workchain >= 128) { - v->error("anycast addresses not supported"); + seen_dot = true; + } else if (c >= '0' && c <= '9') { + if (!seen_dot) { + integer_part = integer_part * 10 + (c - '0'); + if (++integer_digits > 9) { + throw ParseError(loc, "argument is too big and leads to overflow"); + } + } else if (fractional_digits < 9) { + fractional_part = fractional_part * 10 + (c - '0'); + fractional_digits++; } - - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); - td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); - td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return td::BitSlice{data, sizeof(data)}.to_hex(); + } else { + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } - default: - tolk_assert(false); } + + while (fractional_digits < 9) { // after "0.05" fractional_digits is 2, scale up to 9 + fractional_part *= 10; + fractional_digits++; + } + + int64_t result = integer_part * 1000000000LL + fractional_part; + return td::make_refint(is_negative ? -result : result); } -static td::RefInt256 parse_vertex_string_const_as_int(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 'u': { - td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(str)); - if (str.empty()) { - v->error("empty integer ascii-constant"); - } - if (intval.is_null()) { - v->error("too long integer ascii-constant"); +// given `ton("0.05")` evaluate it to 50000000 +// given `stringCrc32("some_str")` evaluate it +// etc. +static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { + // most functions accept 1 argument, but static compile-time methods like `MyStruct.getDeclaredPackPrefix()` have 0 args + if (v->get_num_args() == 0) { + TypePtr receiver = v->fun_maybe->receiver_type; + f_name = v->fun_maybe->method_name; + + if (f_name == "getDeclaredPackPrefix" || f_name == "getDeclaredPackPrefixLen") { + const TypeDataStruct* t_struct = receiver->try_as(); + if (!t_struct || !t_struct->struct_ref->opcode.exists()) { + throw ParseError(v->loc, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); } - return intval; - } - case 'h': - case 'H': { - unsigned char hash[32]; - digest::hash_str(hash, str.data(), str.size()); - return td::bits_to_refint(hash, (v->modifier == 'h') ? 32 : 256, false); + return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); } - case 'c': { - return td::make_refint(td::crc32(td::Slice{str})); - } - default: - tolk_assert(false); } -} + tolk_assert(v->get_num_args() == 1); // checked by type inferring + AnyExprV v_arg = v->get_arg(0)->get_expr(); -struct ConstantEvaluator { - static bool is_overflow(const td::RefInt256& intval) { - return intval.is_null() || !intval->signed_fits_bits(257); + std::string_view str; + if (auto as_string = v_arg->try_as()) { + str = as_string->str_val; + } else { + // ton(SOME_CONST) is not supported + // ton(0.05) is not supported (it can't be represented in AST even) + // stringCrc32(SOME_CONST) / stringCrc32(some_var) also, it's compile-time literal-only + } + if (str.empty()) { + fire_error_const_string_required(v->loc, f_name, f_name == "ton" ? "0.05" : "some_str"); } - static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { - if (!rhs.is_int()) { - v->error("invalid operator, expecting integer"); - } - td::RefInt256 intval = std::get(rhs.value); - - switch (v->tok) { - case tok_minus: - intval = -intval; - break; - case tok_plus: - break; - case tok_bitwise_not: - intval = ~intval; - break; - case tok_logical_not: - intval = td::make_refint(intval == 0 ? -1 : 0); - break; - default: - v->error("not a constant expression"); - } + if (f_name == "ton") { + return parse_nanotons_as_floating_string(v_arg->loc, str); + } + + if (f_name == "address") { // previously, postfix "..."a, but it returned `slice` (now returns `address`) + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + parse_any_std_address(str, v_arg->loc, &data); + return td::BitSlice{data, sizeof(data)}.to_hex(); + } + + if (f_name == "stringCrc32") { // previously, postfix "..."c + return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); + } + + if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time + return td::make_refint(td::crc16(td::Slice{str.data(), str.size()})); + } + + if (f_name == "stringSha256") { // previously, postfix "..."H + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return td::bits_to_refint(hash, 256, false); + } + + if (f_name == "stringSha256_32") { // previously, postfix "..."h + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return td::bits_to_refint(hash, 32, false); + } - if (is_overflow(intval)) { - v->error("integer overflow"); + if (f_name == "stringHexToSlice") { // previously, postfix "..."s + unsigned char buff[128]; + long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + v_arg->error("invalid hex bitstring constant"); } - return ConstantValue::from_int(std::move(intval)); + return static_cast(str); } - static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { - if (!lhs.is_int() || !rhs.is_int()) { - v->error("invalid operator, expecting integer"); + if (f_name == "stringToBase256") { // previously, postfix "..."u + td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(td::Slice(str.data(), str.size()))); + if (str.empty()) { + v_arg->error("empty integer ascii-constant"); } - td::RefInt256 lhs_intval = std::get(lhs.value); - td::RefInt256 rhs_intval = std::get(rhs.value); - td::RefInt256 intval; - - switch (v->tok) { - case tok_minus: - intval = lhs_intval - rhs_intval; - break; - case tok_plus: - intval = lhs_intval + rhs_intval; - break; - case tok_mul: - intval = lhs_intval * rhs_intval; - break; - case tok_div: - intval = lhs_intval / rhs_intval; - break; - case tok_mod: - intval = lhs_intval % rhs_intval; - break; - case tok_lshift: - intval = lhs_intval << static_cast(rhs_intval->to_long()); - break; - case tok_rshift: - intval = lhs_intval >> static_cast(rhs_intval->to_long()); - break; - case tok_bitwise_and: - intval = lhs_intval & rhs_intval; - break; - case tok_bitwise_or: - intval = lhs_intval | rhs_intval; - break; - case tok_bitwise_xor: - intval = lhs_intval ^ rhs_intval; - break; - case tok_eq: - intval = td::make_refint(lhs_intval == rhs_intval ? -1 : 0); - break; - case tok_lt: - intval = td::make_refint(lhs_intval < rhs_intval ? -1 : 0); - break; - case tok_gt: - intval = td::make_refint(lhs_intval > rhs_intval ? -1 : 0); - break; - case tok_leq: - intval = td::make_refint(lhs_intval <= rhs_intval ? -1 : 0); - break; - case tok_geq: - intval = td::make_refint(lhs_intval >= rhs_intval ? -1 : 0); - break; - case tok_neq: - intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); - break; - default: - v->error("unsupported binary operator in constant expression"); + if (intval.is_null()) { + v_arg->error("too long integer ascii-constant"); } + return std::move(intval); + } - if (is_overflow(intval)) { - v->error("integer overflow"); - } - return ConstantValue::from_int(std::move(intval)); + tolk_assert(false); +} + + +struct ConstantExpressionChecker { + static void handle_unary_operator(V v) { + visit(v->get_rhs()); } - static ConstantValue handle_reference(V v) { - // todo better handle "appears, directly or indirectly, in its own initializer" - std::string_view name = v->get_name(); - const Symbol* sym = lookup_global_symbol(name); - if (!sym) { - v->error("undefined symbol `" + static_cast(name) + "`"); - } - GlobalConstPtr const_ref = sym->try_as(); + static void handle_binary_operator(V v) { + visit(v->get_lhs()); + visit(v->get_rhs()); + } + + // `const a = 1 + b`, we met `b` + static void handle_reference(V v) { + GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - v->error("symbol `" + static_cast(name) + "` is not a constant"); + v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); } - if (v->has_instantiationTs()) { // SOME_CONST - v->error("constant is not a generic"); + } + + // `const a = ton("0.05")`, we met `ton("0.05")` + static void handle_function_call(V v) { + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { + // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one + return; + } + v->error("not a constant expression"); + } + + // `const a = (1, 2)`, why not; or it's a default value of a field + static void handle_tensor(V v) { + for (int i = 0; i < v->size(); ++i) { + visit(v->get_item(i)); } - return {const_ref->value}; } - static ConstantValue visit(AnyExprV v) { - if (auto v_int = v->try_as()) { - return ConstantValue::from_int(v_int->intval); + // `a: Options = {}`, object literal may occur as a default value for parameters + static void handle_object_literal(V v) { + for (int i = 0; i < v->get_num_fields(); ++i) { + visit(v->get_field(i)->get_init_val()); } - if (auto v_bool = v->try_as()) { - return ConstantValue::from_int(v_bool->bool_val ? -1 : 0); + } + + static void visit(AnyExprV v) { + if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { + return; } if (auto v_unop = v->try_as()) { - return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); + return handle_unary_operator(v_unop); } if (auto v_binop = v->try_as()) { - return handle_binary_operator(v_binop, visit(v_binop->get_lhs()), visit(v_binop->get_rhs())); + return handle_binary_operator(v_binop); } if (auto v_ref = v->try_as()) { return handle_reference(v_ref); } + if (auto v_call = v->try_as()) { + return handle_function_call(v_call); + } + if (auto v_tensor = v->try_as()) { + return handle_tensor(v_tensor); + } + if (auto v_obj = v->try_as()) { + return handle_object_literal(v_obj->get_body()); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } - if (v->try_as()) { - return eval_const_init_value(v); + if (auto v_cast_to = v->try_as()) { + return visit(v_cast_to->get_expr()); + } + if (auto v_dot = v->try_as(); v_dot && (v_dot->is_target_indexed_access() || v_dot->is_target_fun_ref())) { + return visit(v_dot->get_obj()); } v->error("not a constant expression"); } - static ConstantValue eval_const_init_value(AnyExprV init_value) { - // it init_value is incorrect, an exception is thrown - return visit(init_value); + // check that `2 + 3` is constant + // type inferring has already passed, so types are correct, `1 + ""` can not occur + // if v is not a constant expression like `foo()`, an exception is thrown + static void check_expression_expected_to_be_constant(AnyExprV v) { + visit(v); } }; -ConstantValue eval_const_init_value(AnyExprV init_value) { - // at first, handle most simple cases, not to launch heavy computation algorithm: just a number, just a string - // just `c = 1` or `c = 0xFF` - if (auto v_int = init_value->try_as()) { - return {v_int->intval}; - } - // just `c = "strval"`, probably with modifier (address, etc.) - if (auto v_string = init_value->try_as()) { - if (v_string->is_bitslice()) { - return {parse_vertex_string_const_as_slice(v_string)}; - } else { - return {parse_vertex_string_const_as_int(v_string)}; - } - } - // something more complex, like `c = anotherC` or `c = 1 << 8` - return ConstantEvaluator::eval_const_init_value(init_value); +void check_expression_is_constant(AnyExprV v_expr) { + ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); +} + +std::string eval_string_const_standalone(AnyExprV v_string) { + auto v = v_string->try_as(); + tolk_assert(v); + td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); + return td::hex_encode(str_slice); +} + +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { + auto v = v_call->try_as(); + tolk_assert(v && v->fun_maybe->is_compile_time_const_val()); + return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } + } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 0f99867d8..b7a9fc7fe 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,24 +22,10 @@ namespace tolk { -struct ConstantValue { - std::variant value; +typedef std::variant CompileTimeFunctionResult; - bool is_int() const { return std::holds_alternative(value); } - bool is_slice() const { return std::holds_alternative(value); } - - td::RefInt256 as_int() const { return std::get(value); } - const std::string& as_slice() const { return std::get(value); } - - static ConstantValue from_int(int value) { - return {td::make_refint(value)}; - } - - static ConstantValue from_int(td::RefInt256 value) { - return {std::move(value)}; - } -}; - -ConstantValue eval_const_init_value(AnyExprV init_value); +void check_expression_is_constant(AnyExprV v_expr); +std::string eval_string_const_standalone(AnyExprV v_string); +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); } // namespace tolk diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 8d3b24a8f..f46def4ac 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -19,10 +19,12 @@ namespace tolk { struct ASTNodeBase; +struct ASTNodeDeclaredTypeBase; struct ASTNodeExpressionBase; struct ASTNodeStatementBase; using AnyV = const ASTNodeBase*; +using AnyTypeV = const ASTNodeDeclaredTypeBase*; using AnyExprV = const ASTNodeExpressionBase*; using AnyStatementV = const ASTNodeStatementBase*; @@ -31,15 +33,33 @@ struct LocalVarData; struct FunctionData; struct GlobalVarData; struct GlobalConstData; +struct AliasDefData; +struct StructFieldData; +struct StructData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; using GlobalVarPtr = const GlobalVarData*; using GlobalConstPtr = const GlobalConstData*; +using AliasDefPtr = const AliasDefData*; +using StructFieldPtr = const StructFieldData*; +using StructPtr = const StructData*; class TypeData; using TypePtr = const TypeData*; +struct GenericsSubstitutions; + struct SrcFile; +enum class FunctionInlineMode { + notCalculated, + inlineViaFif, + inlineRef, + inlineInPlace, + noInline, +}; + +typedef int var_idx_t; + } // namespace tolk diff --git a/tolk/gen-entrypoints.cpp b/tolk/gen-entrypoints.cpp new file mode 100644 index 000000000..196b3dd16 --- /dev/null +++ b/tolk/gen-entrypoints.cpp @@ -0,0 +1,120 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "gen-entrypoints.h" +#include "type-system.h" + +/* + * This module is responsible for `onInternalMessage` at IR generation. + * In FunC, `recv_internal()` entrypoint was declared in such a way: + * > () recv_internal(int my_balance, int msg_value, cell in_msg_full, slice in_msg_body) + * Whenever the user wanted to check whether the message is bounced, he had to parse the cell manually. + * + * In Tolk: + * > fun onInternalMessage(in: InMessage) + * And to use `in.senderAddress`, `in.body`, `in.originalForwardFee`, etc. in the function. + * Under the hood, `in.senderAddress` is transformed into `INMSG_SRC`, and so on. + * + * Also, if `onBouncedMessage` exists, it's embedded directly, like + * > if (INMSG_BOUNCED) { onBouncedMessage(in.body); return; } + */ + +namespace tolk { + +// implemented in ast-from-legacy.cpp +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); + + +// check for "modern" `fun onInternalMessage(in: InMessage)`, +// because a "FunC-style" (msgCell: cell, msgBody: slice) also works; +// after transformation `in.xxx` to TVM aux vertices, the last parameter is named `in.body` +static bool is_modern_onInternalMessage(FunctionPtr f_onInternalMessage) { + return f_onInternalMessage->get_num_params() == 1 && f_onInternalMessage->get_param(0).name == "in.body"; +} + +std::vector AuxData_OnInternalMessage_getField::generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const { + if (field_name == "body") { // take `in.body` from a stack + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + if (field_name == "bouncedBody") { // we're in onBouncedMessage() + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + + int idx = -1; + if (field_name == "isBounced") idx = 1; + else if (field_name == "senderAddress") idx = 2; + else if (field_name == "originalForwardFee") idx = 3; + else if (field_name == "createdLt") idx = 4; + else if (field_name == "createdAt") idx = 5; + else if (field_name == "valueCoins") idx = 7; + else if (field_name == "valueExtra") idx = 8; + tolk_assert(idx != -1); + + std::vector ir_msgparam = code.create_tmp_var(TypeDataInt::create(), loc, field_name.data()); + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{code.create_int(loc, idx, "(param-idx)")}, lookup_function("__InMessage.getInMsgParam")); + + if (field_name == "originalForwardFee") { + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{ir_msgparam[0], code.create_int(loc, 0, "(basechain)")}, lookup_function("__InMessage.originalForwardFee")); + } + + return ir_msgparam; +} + +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc) { + // ignore FunC-style `onInternalMessage(msgCell, msgBody)` + if (!is_modern_onInternalMessage(f_onInternalMessage)) { + return; + } + // ignore `@on_bounced_policy("manual")`, don't insert "if (isBounced) return" + if (f_onInternalMessage->is_manual_on_bounce()) { + return; + } + + const Symbol* sym = lookup_global_symbol("onBouncedMessage"); + FunctionPtr f_onBouncedMessage = sym ? sym->try_as() : nullptr; + + AuxData_OnInternalMessage_getField get_isBounced(f_onInternalMessage, "isBounced"); + std::vector ir_isBounced = get_isBounced.generate_get_InMessage_field(code, loc); + + if (f_onBouncedMessage) { + // generate: `if (isBounced) { onBouncedMessage(); return; } + tolk_assert(f_onBouncedMessage->inferred_return_type->get_width_on_stack() == 0); + Op& if_isBounced = code.emplace_back(loc, Op::_If, ir_isBounced); + { + code.push_set_cur(if_isBounced.block0); + std::vector ir_bodySlice(rvect_params.end() - 1, rvect_params.end()); + if (f_onBouncedMessage->is_inlined_in_place()) { + gen_inline_fun_call_in_place(code, TypeDataVoid::create(), loc, f_onBouncedMessage, nullptr, true, {ir_bodySlice}); + } else { + Op& op_call = code.emplace_back(loc, Op::_Call, std::vector{}, ir_bodySlice, f_onBouncedMessage); + op_call.set_impure_flag(); + } + code.emplace_back(loc, Op::_Return, std::vector{}); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_isBounced.block1); + code.close_pop_cur(loc); + } + } else { + // generate: `assert (!isBounced) throw 0` + std::vector args_throw0if = { code.create_int(loc, 0, "(exit-0)"), ir_isBounced[0], code.create_int(loc, 1, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } +} + +} // namespace tolk diff --git a/tolk/ast-from-tokens.h b/tolk/gen-entrypoints.h similarity index 70% rename from tolk/ast-from-tokens.h rename to tolk/gen-entrypoints.h index 39574f9c2..90f62b9bc 100644 --- a/tolk/ast-from-tokens.h +++ b/tolk/gen-entrypoints.h @@ -16,10 +16,13 @@ */ #pragma once +#include "ast-aux-data.h" #include "fwd-declarations.h" +#include "tolk.h" namespace tolk { -AnyV parse_src_file_to_ast(const SrcFile* file); +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc); +std::vector generate_get_requested_field_parsing_on_demand(const AuxData_OnInternalMessage_getField* aux_data, CodeBlob& code, SrcLocation loc); } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 9dae3f009..ae51cf752 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -24,49 +24,106 @@ namespace tolk { -// given orig = "(int, T)" and substitutions = [slice], return "(int, slice)" -static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsDeclaration* genericTs, const std::vector& substitutionTs) { +// given orig `(int, T)` and substitutions [slice], return `(int, slice)` +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs, bool apply_defaultTs = false, std::string_view* out_unknownT = nullptr) { if (!orig || !orig->has_genericT_inside()) { return orig; } - tolk_assert(genericTs->size() == substitutionTs.size()); - return orig->replace_children_custom([genericTs, substitutionTs](TypePtr child) { + return orig->replace_children_custom([substitutedTs, apply_defaultTs, &out_unknownT](TypePtr child) { if (const TypeDataGenericT* asT = child->try_as()) { - int idx = genericTs->find_nameT(asT->nameT); - if (idx == -1) { - throw Fatal("can not replace generic " + asT->nameT); + TypePtr typeT = substitutedTs->get_substitution_for_nameT(asT->nameT); + if (typeT == nullptr && apply_defaultTs) { + typeT = substitutedTs->get_default_for_nameT(asT->nameT); } - if (substitutionTs[idx] == nullptr) { - throw GenericDeduceError("can not deduce " + asT->nameT); + if (typeT == nullptr) { // T was not deduced yet, leave T as generic + typeT = child; + if (out_unknownT && out_unknownT->empty()) { + *out_unknownT = asT->nameT; + } + } + return typeT; + } + if (const TypeDataGenericTypeWithTs* as_instTs = child->try_as(); as_instTs && !as_instTs->has_genericT_inside()) { + if (StructPtr struct_ref = as_instTs->struct_ref) { + struct_ref = instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, as_instTs->type_arguments)); + return TypeDataStruct::create(struct_ref); + } + if (AliasDefPtr alias_ref = as_instTs->alias_ref) { + alias_ref = instantiate_generic_alias(as_instTs->alias_ref, GenericsSubstitutions(alias_ref->genericTs, as_instTs->type_arguments)); + return TypeDataAlias::create(alias_ref); } - return substitutionTs[idx]; } return child; }); } -GenericSubstitutionsDeduceForCall::GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref) - : fun_ref(fun_ref) { - substitutionTs.resize(fun_ref->genericTs->size()); // filled with nullptr (nothing deduced) +GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) + : genericTs(genericTs) + , valuesTs(genericTs->size()) { + provide_type_arguments(type_arguments); } -void GenericSubstitutionsDeduceForCall::provide_deducedT(const std::string& nameT, TypePtr deduced) { - if (deduced == TypeDataNullLiteral::create() || deduced->has_unknown_inside()) { - return; // just 'null' doesn't give sensible info +std::string GenericsSubstitutions::as_human_readable(bool show_nullptr) const { + std::string result; + for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr && !show_nullptr) { + continue;; + } + if (!result.empty()) { + result += ", "; + } + result += genericTs->get_nameT(i); + if (valuesTs[i] == nullptr) { + result += "=nullptr"; + } else { + result += "=`"; + result += valuesTs[i]->as_human_readable(); + result += "`"; + } } + return result; +} - int idx = fun_ref->genericTs->find_nameT(nameT); - if (substitutionTs[idx] == nullptr) { - substitutionTs[idx] = deduced; - } else if (substitutionTs[idx] != deduced) { - throw GenericDeduceError(nameT + " is both " + substitutionTs[idx]->as_human_readable() + " and " + deduced->as_human_readable()); +void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { + for (int i = 0; i < size(); ++i) { + if (genericTs->get_nameT(i) == nameT) { + if (valuesTs[i] == nullptr) { + tolk_assert(!typeT->has_genericT_inside()); + valuesTs[i] = typeT; + } + break; + } } } -void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector&& substitutionTs) { - this->substitutionTs = std::move(substitutionTs); - this->manually_specified = true; +void GenericsSubstitutions::provide_type_arguments(const std::vector& type_arguments) { + tolk_assert(genericTs != nullptr); + int start_from = genericTs->n_from_receiver; // for `Container.wrap` user should specify only U + tolk_assert(static_cast(type_arguments.size()) + start_from == genericTs->size()); + for (int i = start_from; i < genericTs->size(); ++i) { + valuesTs[i] = type_arguments[i - start_from]; + } +} + +void GenericsSubstitutions::rewrite_missing_with_defaults() { + for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr) { + valuesTs[i] = genericTs->get_defaultT(i); // if no default, left nullptr + } + } +} + +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(FunctionPtr fun_ref) + : fun_ref(fun_ref) + , struct_ref(nullptr) + , deducedTs(fun_ref->genericTs) { +} + +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(StructPtr struct_ref) + : fun_ref(nullptr) + , struct_ref(struct_ref) + , deducedTs(struct_ref->genericTs) { } // purpose: having `f(value: T)` and call `f(5)`, deduce T = int @@ -75,114 +132,153 @@ void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector(a: int, b: T1, c: (T1, T2))` and call `f(6, 7, (8, cs))` // - `a` does not affect, it doesn't depend on generic Ts // - next condition: param_type = `T1`, arg_type = `int`, deduce T1 = int -// - next condition: param_type = `(T1, T2)`, arg_type = `(int, slice)`, deduce T1 = int, T2 = slice -// for call `f(6, cs, (8, cs))` T1 will be both `slice` and `int`, fired an error -void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_type, TypePtr arg_type) { +// - next condition: param_type = `(T1, T2)` = `(int, T2)`, arg_type = `(int, slice)`, deduce T2 = slice +// for call `f(6, cs, (8, cs))` both T1 and T2 will become `slice`, firing a type mismatch error later +void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, TypePtr arg_type) { + // all Ts deduced up to this point are apriori + param_type = replace_genericT_with_deduced(param_type, &deducedTs); + if (!param_type->has_genericT_inside()) { + return; + } + if (const auto* asT = param_type->try_as()) { // `(arg: T)` called as `f([1, 2])` => T is [int, int] - provide_deducedT(asT->nameT, arg_type); - } else if (const auto* p_nullable = param_type->try_as()) { + deducedTs.set_typeT(asT->nameT, arg_type); + } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->try_as()) { - consider_next_condition(p_nullable->inner, a_nullable->inner); + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { + consider_next_condition(p_nullable->or_null, a_nullable->or_null); } // `arg: T?` called as `f(int)` => T is int else { - consider_next_condition(p_nullable->inner, arg_type); + consider_next_condition(p_nullable->or_null, arg_type); } } else if (const auto* p_tensor = param_type->try_as()) { // `arg: (int, T)` called as `f((5, cs))` => T is slice - if (const auto* a_tensor = arg_type->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { + if (const auto* a_tensor = arg_type->unwrap_alias()->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { for (int i = 0; i < a_tensor->size(); ++i) { consider_next_condition(p_tensor->items[i], a_tensor->items[i]); } } - } else if (const auto* p_tuple = param_type->try_as()) { + } else if (const auto* p_tuple = param_type->try_as()) { // `arg: [int, T]` called as `f([5, cs])` => T is slice - if (const auto* a_tuple = arg_type->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { for (int i = 0; i < a_tuple->size(); ++i) { consider_next_condition(p_tuple->items[i], a_tuple->items[i]); } } } else if (const auto* p_callable = param_type->try_as()) { // `arg: fun(TArg) -> TResult` called as `f(calcTupleLen)` => TArg is tuple, TResult is int - if (const auto* a_callable = arg_type->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { + if (const auto* a_callable = arg_type->unwrap_alias()->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { for (int i = 0; i < a_callable->params_size(); ++i) { consider_next_condition(p_callable->params_types[i], a_callable->params_types[i]); } consider_next_condition(p_callable->return_type, a_callable->return_type); } + } else if (const auto* p_union = param_type->try_as()) { + // `arg: T1 | T2` called as `f(intOrBuilder)` => T1 is int, T2 is builder + // `arg: int | T1` called as `f(builderOrIntOrSlice)` => T1 is builder|slice + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + std::vector p_generic; + std::vector a_sub_p = a_union->variants; + bool is_sub_correct = true; + for (TypePtr p_variant : p_union->variants) { + if (!p_variant->has_genericT_inside()) { + auto it = std::find(a_sub_p.begin(), a_sub_p.end(), p_variant); + if (it != a_sub_p.end()) { + a_sub_p.erase(it); + } else { + is_sub_correct = false; + } + } else { + p_generic.push_back(p_variant); + } + } + if (is_sub_correct && p_generic.size() == 1 && a_sub_p.size() > 1) { + consider_next_condition(p_generic[0], TypeDataUnion::create(std::move(a_sub_p))); + } else if (is_sub_correct && p_generic.size() == a_sub_p.size()) { + for (int i = 0; i < static_cast(p_generic.size()); ++i) { + consider_next_condition(p_generic[i], a_sub_p[i]); + } + } + } + // `arg: int | MyData` called as `f(MyData)` => T is int + else { + for (TypePtr p_variant : p_union->variants) { + consider_next_condition(p_variant, arg_type); + } + } + } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { + // `arg: Wrapper` called as `f(wrappedInt)` => T is int + if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + tolk_assert(p_instSt->size() == a_struct->struct_ref->substitutedTs->size()); + for (int i = 0; i < p_instSt->size(); ++i) { + consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); + } + } + } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { + // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int + if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { + tolk_assert(p_instAl->size() == a_alias->alias_ref->substitutedTs->size()); + for (int i = 0; i < p_instAl->size(); ++i) { + consider_next_condition(p_instAl->type_arguments[i], a_alias->alias_ref->substitutedTs->typeT_at(i)); + } + } } } -TypePtr GenericSubstitutionsDeduceForCall::replace_by_manually_specified(TypePtr param_type) const { - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); +TypePtr GenericSubstitutionsDeducing::replace_Ts_with_currently_deduced(TypePtr orig) const { + return replace_genericT_with_deduced(orig, &deducedTs); } -TypePtr GenericSubstitutionsDeduceForCall::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { - try { - if (!manually_specified) { - consider_next_condition(param_type, arg_type); - } - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); - } catch (const GenericDeduceError& ex) { - throw ParseError(cur_f, loc, ex.message + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); - } +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type) { + consider_next_condition(param_type, arg_type); + return replace_genericT_with_deduced(param_type, &deducedTs); } -int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i] == nullptr) { - return i; - } +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { + std::string_view unknown_nameT; + consider_next_condition(param_type, arg_type); + param_type = replace_genericT_with_deduced(param_type, &deducedTs, true, &unknown_nameT); + if (param_type->has_genericT_inside()) { + fire_error_can_not_deduce(cur_f, loc, unknown_nameT); } - return -1; + return param_type; } -// clone the body of `f` replacing T everywhere with a substitution -// before: `fun f(v: T) { var cp: [T] = [v]; }` -// after: `fun f(v: int) { var cp: [int] = [v]; }` -// an instantiated function becomes a deep copy, all AST nodes are copied, no previous pointers left -class GenericFunctionReplicator final : public ASTReplicatorFunction { - const GenericsDeclaration* genericTs; - const std::vector& substitutionTs; - -protected: - using ASTReplicatorFunction::clone; - - TypePtr clone(TypePtr t) override { - return replace_genericT_with_deduced(t, genericTs, substitutionTs); +std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() const { + for (int i = 0; i < deducedTs.size(); ++i) { + if (deducedTs.typeT_at(i) == nullptr) { + return deducedTs.nameT_at(i); + } } + return ""; +} -public: - GenericFunctionReplicator(const GenericsDeclaration* genericTs, const std::vector& substitutionTs) - : genericTs(genericTs) - , substitutionTs(substitutionTs) { - } +void GenericSubstitutionsDeducing::apply_defaults_from_declaration() { + deducedTs.rewrite_missing_with_defaults(); +} - V clone_function_body(V v_function) override { - return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()), - clone(v_function->declared_return_type), - nullptr, // a newly-created function is not generic - v_function->method_id, - v_function->flags - ); +void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { + if (fun_ref) { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + } else { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); } -}; +} -std::string GenericsDeclaration::as_human_readable() const { - std::string result = "<"; - for (const GenericsItem& item : itemsT) { - if (result.size() > 1) { - result += ","; +std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) const { + std::string result; + if (int start_from = include_from_receiver ? 0 : n_from_receiver; start_from < size()) { + result += '<'; + for (int i = start_from; i < size(); ++i) { + if (result.size() > 1) { + result += ", "; + } + result += itemsT[i].nameT; } - result += item.nameT; + result += '>'; } - result += ">"; return result; } @@ -195,77 +291,237 @@ int GenericsDeclaration::find_nameT(std::string_view nameT) const { return -1; } -// after creating a deep copy of `f` like `f`, its new and fresh body needs the previous pipeline to run -// for example, all local vars need to be registered as symbols, etc. -static void run_pipeline_for_instantiated_function(FunctionPtr inst_fun_ref) { - // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring - pipeline_resolve_identifiers_and_assign_symbols(inst_fun_ref); - pipeline_calculate_rvalue_lvalue(inst_fun_ref); - pipeline_infer_types_and_calls_and_fields(inst_fun_ref); +bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { + return genericTs->find_nameT(nameT) != -1; +} + +TypePtr GenericsSubstitutions::get_substitution_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : valuesTs[idx]; } -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { +TypePtr GenericsSubstitutions::get_default_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : genericTs->get_defaultT(idx); +} + +// given this= and rhs=, check that T1 is equal to T2 in terms of "equal_to" of TypePtr +// for example, `Wrapper>` / `Wrapper>` / `Wrapper` are equal +bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { + if (size() != rhs->size()) { + return false; + } + for (int i = 0; i < size(); ++i) { + if (!valuesTs[i]->equal_to(rhs->valuesTs[i])) { + return false; + } + } + return true; +} + +// when cloning `f`, original name is "f", we need a new name for symtable and output +// name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) +static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces, int size_from_receiver = 0) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; - name += "<"; - for (TypePtr subs : substitutions) { - if (name.size() > orig_name.size() + 1) { - name += ","; + if (size_from_receiver < substitutedTs.size()) { + name += '<'; + for (int i = size_from_receiver; i < substitutedTs.size(); ++i) { + if (name.size() > orig_name.size() + 1) { + name += ", "; + } + name += substitutedTs.typeT_at(i)->as_human_readable(); } - name += subs->as_human_readable(); + name += '>'; + } + if (!allow_spaces) { + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); } - name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); - name += ">"; return name; } -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs) { - tolk_assert(fun_ref->genericTs); +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(fun_ref->is_generic_function() && !fun_ref->has_tvm_method_id()); - // if `f` was earlier instantiated, return it - if (const auto* existing = lookup_global_symbol(inst_name)) { - FunctionPtr inst_ref = existing->try_as(); - tolk_assert(inst_ref); - return inst_ref; + // fun_ref->name = "f", inst_name will be "f" and similar + std::string fun_name = fun_ref->name; + if (fun_ref->is_method() && fun_ref->receiver_type->has_genericT_inside()) { + fun_name = replace_genericT_with_deduced(fun_ref->receiver_type, &substitutedTs)->as_human_readable() + "." + fun_ref->method_name; + } + std::string new_name = generate_instantiated_name(fun_name, substitutedTs, false, fun_ref->genericTs->n_from_receiver); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + FunctionPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; } - std::vector parameters; - parameters.reserve(fun_ref->get_num_params()); - for (const LocalVarData& orig_p : fun_ref->parameters) { - parameters.emplace_back(orig_p.name, orig_p.loc, replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, substitutionTs), orig_p.flags, orig_p.param_idx); + // to store permanently, allocate an object in heap + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + + // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. + // nevertheless, for outer code to follow a single algorithm, + // when calling `debugPrint(x)`, we clone it as "debugPrint", replace types, and insert into symtable + if (fun_ref->is_builtin_function()) { + std::vector new_parameters; + new_parameters.reserve(fun_ref->get_num_params()); + for (const LocalVarData& orig_p : fun_ref->parameters) { + TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); + new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.default_value, orig_p.flags, orig_p.param_idx); + } + TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); + TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, fun_ref->inline_mode, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + new_fun_ref->arg_order = fun_ref->arg_order; + new_fun_ref->ret_order = fun_ref->ret_order; + new_fun_ref->base_fun_ref = fun_ref; + G.symtable.add_function(new_fun_ref); + return new_fun_ref; } - TypePtr declared_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, substitutionTs); - const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); - - if (fun_ref->is_asm_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyAsm, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; + + // for `f` (both asm and regular), create "f" with AST fully cloned + // it means, that types still contain T: `f(v: T): T`, but since type resolving knows + // it's instantiation, when resolving types, it substitutes T=int + V orig_root = fun_ref->ast_root->as(); + V new_root = ASTReplicator::clone_function_ast(orig_root); + + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, std::move(new_name), allocatedTs); + tolk_assert(new_fun_ref); + // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run + // for example, all local vars need to be registered as symbols, etc. + // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring + pipeline_resolve_identifiers_and_assign_symbols(new_fun_ref); + pipeline_resolve_types_and_aliases(new_fun_ref); + pipeline_calculate_rvalue_lvalue(new_fun_ref); + pipeline_infer_types_and_calls_and_fields(new_fun_ref); + + return new_fun_ref; +} + +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(struct_ref->is_generic_struct()); + + // if `Wrapper` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(struct_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + StructPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; } - if (fun_ref->is_builtin_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - return inst_ref; + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = struct_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); + + StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, std::move(new_name), allocatedTs); + tolk_assert(new_struct_ref); + pipeline_resolve_identifiers_and_assign_symbols(new_struct_ref); + pipeline_resolve_types_and_aliases(new_struct_ref); + return new_struct_ref; +} + +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(alias_ref->is_generic_alias()); + + // if `Response` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(alias_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + AliasDefPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; } - GenericFunctionReplicator replicator(fun_ref->genericTs, instantiationTs->substitutions); - V inst_root = replicator.clone_function_body(fun_ref->ast_root->as()); - - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyCode, inst_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - inst_root->mutate()->assign_fun_ref(inst_ref); - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = alias_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); + + AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, std::move(new_name), allocatedTs); + tolk_assert(new_alias_ref); + pipeline_resolve_types_and_aliases(new_alias_ref); + return new_alias_ref; +} + +// find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" +// most practical case, when a direct method for receiver exists; +// note, that having an alias `type WorkchainNum = int` and methods `WorkchainNum.isMasterchain()`, +// it's okay to call `-1.isMasterchain()`, because int equals to any alias; +// currently there is no chance to change this logic, say, `type AssetList = dict` to have separate methods, +// due to smart casts, types merge of control flow rejoin, etc., which immediately become `cell?` +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { + FunctionPtr exact_found = nullptr; + + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + if (exact_found) { + return nullptr; + } + exact_found = method_ref; + } + } + } + + return exact_found; +} + +// find `int?.copy` / `T.copy` for called_receiver = "int" and called_name = "copy" +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name) { + std::vector candidates; + + // step1: find all methods where a receiver equals to provided, e.g. `MInt.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step2: find all methods where a receiver can accept provided, e.g. `int8.copy` / `int?.copy` / `(int|slice).copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->can_rhs_be_assigned(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 3: try to match generic receivers, e.g. `Container.copy` / `(T?|slice).copy` but NOT `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->has_genericT_inside() && !method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 4: try to match `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + return candidates; } } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 5ed245aff..c8f46ff27 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -24,79 +24,99 @@ namespace tolk { // when a function is declared `f`, this "" is represented as this class -// (not at AST, but at symbol storage level) +// - `fun tuple.push(self, value:T)` itemsT = [T] +// - `struct Pair` itemsT = [A,B] +// - `fun Container.compareWith` itemsT = [T,U], n_from_receiver = 1 +// - `fun Pair.createFrom` itemsT = [U,V] +// - `fun Pair.create` itemsT = [A,B], n_from_receiver = 2 +// - `struct Opts` itemsT = [T default_type=never] struct GenericsDeclaration { - struct GenericsItem { + struct ItemT { std::string_view nameT; + TypePtr default_type; // exists for ``, nullptr otherwise - explicit GenericsItem(std::string_view nameT) - : nameT(nameT) {} + ItemT(std::string_view nameT, TypePtr default_type) + : nameT(nameT), default_type(default_type) {} }; - explicit GenericsDeclaration(std::vector&& itemsT) - : itemsT(std::move(itemsT)) {} + explicit GenericsDeclaration(std::vector&& itemsT, int n_from_receiver) + : itemsT(std::move(itemsT)) + , n_from_receiver(n_from_receiver) {} - const std::vector itemsT; + const std::vector itemsT; + const int n_from_receiver; - std::string as_human_readable() const; + std::string as_human_readable(bool include_from_receiver = false) const; - size_t size() const { return itemsT.size(); } - bool has_nameT(std::string_view nameT) const { return find_nameT(nameT) != -1; } + int size() const { return static_cast(itemsT.size()); } int find_nameT(std::string_view nameT) const; - std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } + std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } + TypePtr get_defaultT(int idx) const { return itemsT[idx].default_type; } }; // when a function call is `f()`, this "" is represented as this class -struct GenericsInstantiation { - const std::vector substitutions; // for genericTs - const SrcLocation loc; // first instantiation location +// same for `Wrapper`, "" is substitution +struct GenericsSubstitutions { +private: + const GenericsDeclaration* genericTs; // [T1, T2] + std::vector valuesTs; // [SomeStruct, int] - explicit GenericsInstantiation(SrcLocation loc, std::vector&& substitutions) - : substitutions(std::move(substitutions)) - , loc(loc) { +public: + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs) + : genericTs(genericTs) + , valuesTs(genericTs == nullptr ? 0 : genericTs->size()) { } + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments); + + std::string as_human_readable(bool show_nullptr) const; + + int size() const { return static_cast(valuesTs.size()); } + bool has_nameT(std::string_view nameT) const; + TypePtr get_substitution_for_nameT(std::string_view nameT) const; + TypePtr get_default_for_nameT(std::string_view nameT) const; + std::string_view nameT_at(int idx) const { return genericTs->get_nameT(idx); } + TypePtr typeT_at(int idx) const { return valuesTs.at(idx); } + bool equal_to(const GenericsSubstitutions* rhs) const; + + void set_typeT(std::string_view nameT, TypePtr typeT); + void provide_type_arguments(const std::vector& type_arguments); + void rewrite_missing_with_defaults(); }; // this class helps to deduce Ts on the fly // purpose: having `f(value: T)` and call `f(5)`, deduce T = int // while analyzing a call, arguments are handled one by one, by `auto_deduce_from_argument()` -// this class also handles manually specified substitutions like `f(5)` -class GenericSubstitutionsDeduceForCall { +// note, that manually specified substitutions like `f(5)` are NOT handled by this class, it's not deducing +class GenericSubstitutionsDeducing { FunctionPtr fun_ref; - std::vector substitutionTs; - bool manually_specified = false; + StructPtr struct_ref; + GenericsSubstitutions deducedTs; - void provide_deducedT(const std::string& nameT, TypePtr deduced); void consider_next_condition(TypePtr param_type, TypePtr arg_type); public: - explicit GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref); + explicit GenericSubstitutionsDeducing(FunctionPtr fun_ref); + explicit GenericSubstitutionsDeducing(StructPtr struct_ref); - bool is_manually_specified() const { - return manually_specified; - } - - void provide_manually_specified(std::vector&& substitutionTs); - TypePtr replace_by_manually_specified(TypePtr param_type) const; + TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; + TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); - int get_first_not_deduced_idx() const; + std::string_view get_first_not_deduced_nameT() const; + void apply_defaults_from_declaration(); + void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; - std::vector&& flush() { - return std::move(substitutionTs); + GenericsSubstitutions&& flush() { + return std::move(deducedTs); } }; -struct GenericDeduceError final : std::exception { - std::string message; - explicit GenericDeduceError(std::string message) - : message(std::move(message)) { } +typedef std::pair MethodCallCandidate; - const char* what() const noexcept override { - return message.c_str(); - } -}; +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions); -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs); +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name); +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name); } // namespace tolk diff --git a/tolk/lazy-helpers.cpp b/tolk/lazy-helpers.cpp new file mode 100644 index 000000000..22ef5fff2 --- /dev/null +++ b/tolk/lazy-helpers.cpp @@ -0,0 +1,101 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "lazy-helpers.h" +#include "tolk.h" +#include "type-system.h" + +namespace tolk { + +/* + * This file contains "lazy" state across multiple files. + * They all are used after `lazy` operators have been processed and "load xxx" vertices have been inserted. + * Particularly, while transforming AST to Ops. + * For comments about laziness, see pipe-lazy-load-insertions.cpp. + */ + +LazyVariableLoadedState::LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options) + : declared_type(declared_type) + , ir_slice(std::move(ir_slice)) + , ir_options(std::move(ir_options)) + , loaded_state(declared_type->unwrap_alias()->try_as() ? declared_type->unwrap_alias()->try_as()->struct_ref : nullptr) { + // fill loaded_variants: variants of a lazy union or the last field of a struct if it's a union + const TypeDataUnion* t_union = declared_type->unwrap_alias()->try_as(); + if (is_struct() && loaded_state.original_struct->get_num_fields()) { + t_union = loaded_state.original_struct->fields.back()->declared_type->unwrap_alias()->try_as(); + } + if (t_union) { + variants_state.reserve(t_union->size()); + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + variants_state.emplace_back(variant_struct ? variant_struct->struct_ref : nullptr); + } + } +} + +const LazyStructLoadedState* LazyVariableLoadedState::get_struct_state(StructPtr original_struct) const { + if (loaded_state.original_struct == original_struct) { + return &loaded_state; + } + for (const LazyStructLoadedState& struct_state : variants_state) { + if (struct_state.original_struct == original_struct) { + return &struct_state; + } + } + return nullptr; +} + +void LazyVariableLoadedState::assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const { + // on field access `point.x`, ensure that it's loaded, so the value on a stack is not an occasional null + const LazyStructLoadedState* struct_state = get_struct_state(original_struct); + tolk_assert(struct_state); + tolk_assert(struct_state->was_loaded_once()); + StructFieldPtr hidden_field = struct_state->hidden_struct->find_field(original_field->name); + tolk_assert(hidden_field); + tolk_assert(struct_state->ith_field_was_loaded[hidden_field->field_idx]); +} + +void LazyStructLoadedState::on_started_loading(StructPtr hidden_struct) { + this->hidden_struct = hidden_struct; + this->ith_field_was_loaded.resize(hidden_struct->get_num_fields()); // initially false +} + +void LazyStructLoadedState::on_original_field_loaded(StructFieldPtr hidden_field) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + // for example, `var p = lazy Point; aux "load x"; return p.x`; + // we are at "load x", it exists in Point, here just save it was loaded (for assertions and debugging); + // apart from saving, stack is also updated when loading, `p` becomes `valueX null` +} + +void LazyStructLoadedState::on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + this->aside_gaps_and_tail.emplace_back(hidden_field, std::move(ir_field_gap)); + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "load gap", it does not exist in Storage, so save loaded value separately +} + +std::vector LazyStructLoadedState::get_ir_loaded_aside_field(StructFieldPtr hidden_field) const { + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "st.toCell()" that stores immutable gap before modified "seqno" + for (const auto& [gap_field, ir_field_gap] : this->aside_gaps_and_tail) { + if (gap_field == hidden_field) { + return ir_field_gap; + } + } + tolk_assert(false); +} + +} // namespace tolk diff --git a/tolk/lazy-helpers.h b/tolk/lazy-helpers.h new file mode 100644 index 000000000..8fea95b7b --- /dev/null +++ b/tolk/lazy-helpers.h @@ -0,0 +1,91 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include + +namespace tolk { + +// LazyStructLoadInfo describes how to load a struct: which fields to load, which to skip. +// It's calculated based on variable usages and passed through the pipeline carried by auxiliary AST vertices. +// Based on it, lazy loading Ops are generated in pack-unpack api. +// To understand `hidden_struct`, read pipe-lazy-load-insertions.cpp. +struct LazyStructLoadInfo { + enum ActionWithField { + LoadField, + SkipField, + LazyMatchField, + SaveImmutableTail, + }; + + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_action; // each for corresponding field of a struct + + LazyStructLoadInfo(StructPtr original_struct, StructPtr hidden_struct, std::vector&& ith_field_action) + : original_struct(original_struct) + , hidden_struct(hidden_struct) + , ith_field_action(std::move(ith_field_action)) { + } +}; + +// LazyStructLoadedState represents state (which fields were already loaded) while generating AST to Ops. +// For example, variable `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` is initially "nothing loaded", +// and after "load x" ith_field_action[0] becomes true (and `p` is updated on a stack and becomes `valueX null`). +struct LazyStructLoadedState { + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct = nullptr; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_was_loaded; // each for corresponding field of hidden_struct + std::vector>> aside_gaps_and_tail; + + explicit LazyStructLoadedState(StructPtr original_struct) + : original_struct(original_struct) {} + + void on_started_loading(StructPtr hidden_struct); + void on_original_field_loaded(StructFieldPtr hidden_field); + void on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap); + + bool was_loaded_once() const { return hidden_struct != nullptr; } + std::vector get_ir_loaded_aside_field(StructFieldPtr hidden_field) const; + + LazyStructLoadedState* mutate() const { return const_cast(this); } +}; + +// LazyVariableLoadedState contains a state of a whole lazy variable while generating AST to Ops. +// For example, `var p = lazy Point.fromSlice(s)` contains one struct. +// But `var msg = lazy MyMsgUnion.fromSlice(s)` contains N variants, each with own state, but common lazy slice `s`. +// When inlining a function, like `p.getX()`, `self` of `getX` also becomes a lazy variable pointing to the same state. +struct LazyVariableLoadedState { + TypePtr declared_type; + std::vector ir_slice; // filled by `lazy` operator + std::vector ir_options; // same, comes from `lazy T.fromSlice(s, options)` + LazyStructLoadedState loaded_state; // for struct: filled; for union: empty + std::vector variants_state; // variants of a lazy union or the last field if it's a union + + LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options); + + bool is_struct() const { return loaded_state.original_struct != nullptr; } + bool is_union() const { return loaded_state.original_struct == nullptr; } + + const LazyStructLoadedState* get_struct_state(StructPtr original_struct) const; + void assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const; +}; + + +} // namespace tolk + diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 06913a5fc..44d166768 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -23,7 +23,6 @@ namespace tolk { // By 'chunk' in lexer I mean a token or a list of tokens parsed simultaneously. // E.g., when we meet "str", ChunkString is called, it emits tok_string. -// E.g., when we meet "str"x, ChunkString emits not only tok_string, but tok_string_modifier. // E.g., when we meet //, ChunkInlineComment is called, it emits nothing (just skips a line). // We store all valid chunks lexers in a prefix tree (LexingTrie), see below. struct ChunkLexerBase { @@ -170,8 +169,8 @@ struct ChunkMultilineComment final : ChunkLexerBase { // A string, starting from " // Note, that there are no escape symbols inside: the purpose of strings in Tolk just doesn't need it. -// After a closing quote, a string modifier may be present, like "Ef8zMzMzMzMzMzMzMzMzMzM0vF"a. -// If present, it emits a separate tok_string_modifier. +// In FunC, a string might have ended with a modifier like `"..."c` +// It's not valid in Tolk, valid is `stringCrc32("...")` struct ChunkString final : ChunkLexerBase { bool parse(Lexer* lex) const override { const char* str_begin = lex->c_str(); @@ -187,12 +186,6 @@ struct ChunkString final : ChunkLexerBase { lex->skip_chars(1); lex->add_token(tok_string_const, str_val); - if (std::isalpha(lex->char_at())) { - std::string_view modifier_val(lex->c_str(), 1); - lex->skip_chars(1); - lex->add_token(tok_string_modifier, modifier_val); - } - return true; } }; @@ -238,27 +231,43 @@ struct ChunkAnnotation final : ChunkLexerBase { // A number, may be a hex one. struct ChunkNumber final : ChunkLexerBase { - bool parse(Lexer* lex) const override { + static bool parse_hex_or_bin(Lexer* lex, bool bin) { const char* str_begin = lex->c_str(); - bool hex = false; - if (lex->char_at() == '0' && lex->char_at(1) == 'x') { - lex->skip_chars(2); - hex = true; - } + lex->skip_chars(2); // 0x / 0b if (lex->is_eof()) { return false; } + while (!lex->is_eof()) { char c = lex->char_at(); - if (c >= '0' && c <= '9') { - lex->skip_chars(1); - continue; - } - if (!hex) { + bool ok = bin + ? c == '0' || c == '1' + : (c >= '0' && c <= '9') || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); + if (!ok) { break; } - c |= 0x20; - if (c < 'a' || c > 'f') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_int_const, str_val); + return true; + } + + bool parse(Lexer* lex) const override { + if (lex->char_at() == '0') { + if (lex->char_at(1) == 'x') { + return parse_hex_or_bin(lex, false); + } + if (lex->char_at(1) == 'b') { + return parse_hex_or_bin(lex, true); + } + } + + const char* str_begin = lex->c_str(); + while (!lex->is_eof()) { + char c = lex->char_at(); + if (c < '0' || c > '9') { break; } lex->skip_chars(1); @@ -328,13 +337,13 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { case 2: if (str == "do") return tok_do; if (str == "if") return tok_if; + if (str == "is") return tok_is; if (str == "as") return tok_as; break; case 3: if (str == "var") return tok_var; if (str == "fun") return tok_fun; if (str == "asm") return tok_asm; - if (str == "get") return tok_get; if (str == "try") return tok_try; if (str == "val") return tok_val; break; @@ -345,11 +354,13 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "self") return tok_self; if (str == "tolk") return tok_tolk; if (str == "type") return tok_type; + if (str == "lazy") return tok_lazy; if (str == "enum") return tok_enum; break; case 5: if (str == "const") return tok_const; if (str == "false") return tok_false; + if (str == "match") return tok_match; if (str == "redef") return tok_redef; if (str == "while") return tok_while; if (str == "break") return tok_break; @@ -512,6 +523,9 @@ struct TolkLanguageGrammar { register_token("|=", 2, tok_set_bitwise_or); register_token("^=", 2, tok_set_bitwise_xor); register_token("->", 2, tok_arrow); + register_token("=>", 2, tok_double_arrow); + register_token("++", 2, tok_double_plus); + register_token("--", 2, tok_double_minus); register_token("<=>", 3, tok_spaceship); register_token("~>>", 3, tok_rshiftR); register_token("^>>", 3, tok_rshiftC); @@ -545,15 +559,6 @@ Lexer::Lexer(const SrcFile* file) next(); } -Lexer::Lexer(std::string_view text) - : file(nullptr) - , p_start(text.data()) - , p_end(p_start + text.size()) - , p_next(p_start) - , location() { - next(); -} - void Lexer::next() { while (cur_token_idx == last_token_idx && !is_eof()) { update_location(); @@ -578,13 +583,20 @@ void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { } Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { - return {p_next, cur_token_idx, cur_token}; + return {p_next, cur_token_idx, cur_token, location}; } void Lexer::restore_position(SavedPositionForLookahead saved) { p_next = saved.p_next; cur_token_idx = last_token_idx = saved.cur_token_idx; cur_token = saved.cur_token; + location = saved.loc; +} + +void Lexer::hack_replace_rshift_with_one_triangle() { + // overcome the `>>` problem when parsing generics, leave only `>` here, see comments at usage + assert(cur_token.type == tok_rshift); + cur_token = Token(tok_gt, ">"); } void Lexer::error(const std::string& err_msg) const { diff --git a/tolk/lexer.h b/tolk/lexer.h index 58bc3640a..63f0e6a30 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -26,7 +26,6 @@ enum TokenType { tok_empty, tok_fun, - tok_get, tok_type, tok_enum, tok_struct, @@ -48,7 +47,6 @@ enum TokenType { tok_int_const, tok_string_const, - tok_string_modifier, tok_true, tok_false, tok_null, @@ -66,6 +64,8 @@ enum TokenType { tok_set_div, tok_mod, tok_set_mod, + tok_double_plus, + tok_double_minus, tok_lshift, tok_set_lshift, tok_rshift, @@ -117,9 +117,13 @@ enum TokenType { tok_assert, tok_if, tok_else, + tok_match, + tok_lazy, tok_arrow, + tok_double_arrow, tok_as, + tok_is, tok_tolk, tok_semver, @@ -162,10 +166,10 @@ class Lexer { const char* p_next = nullptr; int cur_token_idx = 0; Token cur_token; + SrcLocation loc; }; explicit Lexer(const SrcFile* file); - explicit Lexer(std::string_view text); Lexer(const Lexer&) = delete; Lexer &operator=(const Lexer&) = delete; @@ -210,6 +214,7 @@ class Lexer { SavedPositionForLookahead save_parsing_position() const; void restore_position(SavedPositionForLookahead saved); + void hack_replace_rshift_with_one_triangle(); void check(TokenType next_tok, const char* str_expected) const { if (cur_token.type != next_tok) { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 76d756386..93c6e2067 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -125,13 +125,6 @@ void Optimizer::show_right() const { std::cerr << std::endl; } -bool Optimizer::say(std::string str) const { - if (debug_) { - std::cerr << str << std::endl; - } - return true; -} - bool Optimizer::find_const_op(int* op_idx, int cst) { for (int i = 0; i < l2_; i++) { if (op_[i]->is_gconst() && tr_[i].get(0) == cst) { @@ -142,6 +135,293 @@ bool Optimizer::find_const_op(int* op_idx, int cst) { return false; } +// purpose: transform `65535 THROW` to `PUSHINT` + `THROWANY`; +// such a technique allows pushing a number onto a stack just before THROW, even if a variable is created in advance; +// used for `T.fromSlice(s, {code:0xFFFF})`, where `tmp = 0xFFFF` + serialization match + `else throw tmp` is generated; +// but since it's constant, it transforms to (unused 0xFFFF) + ... + else "65535 THROW", unwrapped here +bool Optimizer::detect_rewrite_big_THROW() { + bool is_throw = op_[0]->is_custom() && op_[0]->op.ends_with(" THROW"); + if (!is_throw) { + return false; + } + + std::string_view s_num_throw = op_[0]->op; + size_t sp = s_num_throw.find(' '); + if (sp != s_num_throw.rfind(' ') || s_num_throw[0] < '1' || s_num_throw[0] > '9') { + return false; + } + + std::string s_number(s_num_throw.substr(0, sp)); + uint64_t excno = std::stoul(s_number); + if (excno < 2048) { // "9 THROW" left as is, but "N THROW" where N>=2^11 is invalid for Fift + return false; + } + + p_ = 1; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(excno))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "THROWANY", 1, 0)); + return true; +} + +// purpose 1: for b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "x{...} STSLICECONST" +// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "x{00ff00ff} STSLICECONST" +// (since it works at IR level, it also works for const variables and auto-serialization) +bool Optimizer::detect_rewrite_MY_store_int() { + bool first_my_store = op_[0]->is_custom() && op_[0]->op.starts_with("MY_store_int"); + if (!first_my_store) { + return false; + } + bool first_unsigned = op_[0]->op[12] == 'U'; + + int n_merged = 0; + td::RefInt256 total_number = td::make_refint(0); + int total_len = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_number_len = op_[i]->op; // "MY_store_intU 123 32" + if (!s_op_number_len.starts_with("MY_store_int")) { + break; + } + + size_t sp = s_op_number_len.rfind(' '); + std::string s_number(s_op_number_len.substr(13 + 1, sp - 13 - 1)); + int len = std::stoi(std::string(s_op_number_len.substr(sp + 1))); + + if (total_len + len > (255 + first_unsigned)) { + break; + } + if (total_number != 0) { + total_number <<= len; + } + total_number += td::string_to_int256(s_number); + total_len += len; + n_merged++; + } + + // we do not want to always use STSLICECONST; for example, storing "0" 64-bit via x{00...} is more effective + // for a single operation, but in practice, total bytecode becomes larger, which has a cumulative negative effect; + // here is a heuristic "when to use STSLICECONST, when leave PUSHINT + STUR", based on real contracts measurements + bool use_stsliceconst = total_len <= 32 || (total_len <= 48 && total_number >= 256) || (total_len <= 64 && total_number >= 65536) + || (total_len <= 96 && total_number >= (1ULL<<32)) || (total_number > (1ULL<<62)); + if (!use_stsliceconst) { + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + return true; + } + + p_ = n_merged; + q_ = 1; + + // output "x{...}" or "b{...}" (if length not divisible by 4) + const td::RefInt256 base = td::make_refint(total_len % 4 == 0 ? 16 : 2); + const int s_len = base == 16 ? total_len / 4 : total_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + for (int i = s_len - 1; i >= 0 && total_number != 0; --i) { + result[2 + i] = digits[(total_number % base)->to_long()]; + total_number /= base; + } + + result += " STSLICECONST"; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, result, 0, 1)); + return true; +} + +// purpose: consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 +bool Optimizer::detect_rewrite_MY_skip_bits() { + bool first_my_skip = op_[0]->is_custom() && op_[0]->op.starts_with("MY_skip_bits"); + if (!first_my_skip) { + return false; + } + + int n_merged = 0; + int total_skip_bits = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_len = op_[i]->op; // "MY_skip_bits 32" + if (!s_op_len.starts_with("MY_skip_bits")) { + break; + } + + std::string s_number(s_op_len.substr(s_op_len.find(' ') + 1)); + total_skip_bits += std::stoi(s_number); + n_merged++; + } + + p_ = n_merged; + q_ = 2; + if (total_skip_bits <= 256) { + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_skip_bits) + " LDU")); + oq_[1] = std::make_unique(AsmOp::Pop(op_[0]->loc, 1)); + } else { + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + } + return true; +} + +// pattern `NEWC` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `NEWC` + `32 STU`, it's a bit cheaper +bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); // actually there can be PUSHPOWDEC2, but ok + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::move(op_[0]); + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + +// pattern `N LDU` + `DROP` -> `N PLDU` (common after loading the last field manually or by `lazy`); +// the same for LDI -> PLDI, LDREF -> PLDREF, etc. +bool Optimizer::detect_rewrite_LDxx_DROP() { + bool second_drop = pb_ > 1 && op_[1]->is_pop() && op_[1]->a == 0; + if (!second_drop || !op_[0]->is_custom()) { + return false; + } + + static const char* ends_with[] = { " LDI", " LDU", " LDBITS"}; + static const char* repl_with[] = {" PLDI", " PLDU", " PLDBITS"}; + static const char* equl_to[] = { "LDREF", "LDDICT", "LDOPTREF", "LDSLICEX"}; + static const char* repl_to[] = {"PLDREF", "PLDDICT", "PLDOPTREF", "PLDSLICEX"}; + + std::string_view f = op_[0]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i], 0, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} + +// pattern `SWAP` + `EQUAL` -> `EQUAL` +// and other symmetric operators: NEQ, MUL, etc. +bool Optimizer::detect_rewrite_SWAP_symmetric() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + std::string_view n = op_[1]->op; + bool next_symmetric = n == "EQUAL" || n == "NEQ" || n == "SDEQ" || n == "AND" || n == "OR" + || n == "ADD" || n == "MUL" || n == "MIN" || n == "MAX"; + if (!next_symmetric) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::move(op_[1]); + return true; +} + +// pattern `SWAP` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `ROT` + `32 STU` +bool Optimizer::detect_rewrite_SWAP_PUSH_STUR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::make_unique(AsmOp::BlkSwap(oq_[0]->loc, 1, 2)); // ROT + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + +// pattern `SWAP` + `STSLICER` -> `STSLICE` and vice versa: `SWAP` + `STSLICE` => `STSLICER`; +// same for `STB` / `STREF` / `n STU` / `n STI` +bool Optimizer::detect_rewrite_SWAP_STxxxR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" STU", " STI", " STUR", " STIR"}; + static const char* repl_with[] = {" STUR", " STIR", " STU", " STI"}; + static const char* equl_to[] = {"STSLICE", "STSLICER", "STB", "STBR", "SUB", "SUBR", "STREF", "STREFR", "LESS", "LEQ", "GREATER", "GEQ"}; + static const char* repl_to[] = {"STSLICER", "STSLICE", "STBR", "STB", "SUBR", "SUB", "STREFR", "STREF", "GREATER", "GEQ", "LESS", "LEQ"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} + +// pattern `NOT` + `123 THROWIFNOT` -> `123 THROWIF` (and THROWIF -> THROWIFNOT) +bool Optimizer::detect_rewrite_NOT_THROWIF() { + bool first_not = op_[0]->is_custom() && op_[0]->op == "NOT"; + if (!first_not || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" THROWIF", " THROWIFNOT"}; + static const char* repl_with[] = {" THROWIFNOT", " THROWIF"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); + return true; + } + } + + return false; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -157,7 +437,7 @@ bool Optimizer::rewrite_push_const(int i, int c) { show_left(); oq_[1] = std::move(op_[idx]); oq_[0] = std::move(op_[!idx]); - *oq_[0] = AsmOp::Push(i); + *oq_[0] = AsmOp::Push(oq_[0]->loc, i); show_right(); return true; } @@ -176,7 +456,7 @@ bool Optimizer::rewrite_const_rot(int c) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Custom("ROT", 3, 3); + *oq_[1] = AsmOp::Custom(oq_[0]->loc, "ROT", 3, 3); show_right(); return true; } @@ -195,7 +475,7 @@ bool Optimizer::rewrite_const_pop(int c, int i) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Pop(i); + *oq_[1] = AsmOp::Pop(oq_[0]->loc, i); show_right(); return true; } @@ -553,40 +833,46 @@ bool Optimizer::find_at_least(int pb) { pb_ = pb; // show_stack_transforms(); int i, j, k, l, c; + SrcLocation loc; // for asm ops inserted by optimizer, leave location empty (in fift output, it'll be attached to above) return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || - (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(i, j))) || (is_push(&i) && rewrite(AsmOp::Push(i))) || - (is_pop(&i) && rewrite(AsmOp::Pop(i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(i), AsmOp::Pop(j))) || - (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(i, j), AsmOp::Xchg(k, l))) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || + detect_rewrite_big_THROW() || + detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || + detect_rewrite_LDxx_DROP() || + detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || + detect_rewrite_NOT_THROWIF() || (!(mode_ & 1) && - ((is_rot() && rewrite(AsmOp::Custom("ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom("-ROT", 3, 3))) || - (is_2dup() && rewrite(AsmOp::Custom("2DUP", 2, 4))) || - (is_2swap() && rewrite(AsmOp::Custom("2SWAP", 2, 4))) || - (is_2over() && rewrite(AsmOp::Custom("2OVER", 2, 4))) || - (is_tuck() && rewrite(AsmOp::Custom("TUCK", 2, 3))) || - (is_2drop() && rewrite(AsmOp::Custom("2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(i, j))) || - (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(i, j))) || - (is_push2(&i, &j) && rewrite(AsmOp::Push2(i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(i, j))) || - (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(i))) || - (is_push_rot(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("ROT"))) || - (is_push_rotrev(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("-ROT"))) || - (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(i), AsmOp::Xchg(j, k))) || - (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(i, j))) || - (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(i, j))) || - (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(i, j), AsmOp::BlkDrop(i))) || - (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(i), AsmOp::BlkDrop(k))) || + ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom(loc, "2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom(loc, "2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom(loc, "TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom(loc, "2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(loc, i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(loc, i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(loc, i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(loc, i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(loc, i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(loc, i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(loc, i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(loc, i), AsmOp::Xchg(loc, j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(loc, i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(loc, i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::BlkDrop(loc, i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(loc, i), AsmOp::BlkDrop(loc, k))) || (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 - ? rewrite(AsmOp::Xchg2(j + 1, i), AsmOp::BlkDrop(k + 2)) - : rewrite(AsmOp::Pop(i), AsmOp::Pop(j), AsmOp::BlkDrop(k)))) || - (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(i, j, k))) || - (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(i, j, k))) || - (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(i, j, k))) || - (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(i, j, k))) || - (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(i, j, k))) || - (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(i, j, k))) || - (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(i, j, k))) || - (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(i, j, k))))); + ? rewrite(AsmOp::Xchg2(loc, j + 1, i), AsmOp::BlkDrop(loc, k + 2)) + : rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j), AsmOp::BlkDrop(loc, k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(loc, i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(loc, i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(loc, i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(loc, i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(loc, i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(loc, i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(loc, i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(loc, i, j, k))))); } bool Optimizer::find() { diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp new file mode 100644 index 000000000..abab637e9 --- /dev/null +++ b/tolk/pack-unpack-api.cpp @@ -0,0 +1,448 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "lazy-helpers.h" +#include "type-system.h" +#include + +/* + * This module provides high-level (de)serialization functions to be used from outer code: + * - pack to cell/builder + * - unpack from cell/slice + * - etc. + * + * For the implementation of packing primitives, consider `pack-unpack-serializers.cpp`. + */ + +namespace tolk { + + +// -------------------------------------------- +// checking serialization availability +// +// for every call `obj.toCell()` and similar, checks are executed to ensure that `obj` can be serialized +// if it can, the compilation process continues +// if not, a detailed explanation is shown +// + + +struct CantSerializeBecause { + std::string because_msg; + + explicit CantSerializeBecause(std::string because_msg) + : because_msg(std::move(because_msg)) {} + explicit CantSerializeBecause(const std::string& because_msg, const CantSerializeBecause& why) + : because_msg(because_msg + "\n" + why.because_msg) {} +}; + +class PackUnpackAvailabilityChecker { + std::vector already_checked; + + static bool check_declared_packToBuilder(TypePtr receiver_type, FunctionPtr f_pack) { + if (!f_pack->does_accept_self() || f_pack->does_mutate_self() || f_pack->get_num_params() != 2) { + return false; + } + if (f_pack->get_param(1).declared_type != TypeDataBuilder::create() || !f_pack->has_mutate_params()) { + return false; + } + return f_pack->inferred_return_type->get_width_on_stack() == 0; + } + + static bool check_declared_unpackFromSlice(TypePtr receiver_type, FunctionPtr f_unpack) { + if (f_unpack->does_accept_self() || f_unpack->get_num_params() != 1) { + return false; + } + if (f_unpack->get_param(0).declared_type != TypeDataSlice::create() || !f_unpack->has_mutate_params()) { + return false; + } + return f_unpack->inferred_return_type->equal_to(receiver_type); + } + +public: + std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { + if (any_type->try_as()) { + return {}; + } + if (any_type->try_as()) { + return {}; + } + if (any_type == TypeDataCoins::create()) { + return {}; + } + if (any_type == TypeDataBool::create()) { + return {}; + } + if (any_type == TypeDataCell::create()) { + return {}; + } + if (any_type == TypeDataAddress::create()) { + return {}; + } + if (any_type == TypeDataNever::create()) { + return {}; + } + + if (const auto* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + if (std::find(already_checked.begin(), already_checked.end(), struct_ref) != already_checked.end()) { + return {}; + } + already_checked.push_back(struct_ref); // prevent recursion and visiting one struct multiple times + + for (StructFieldPtr field_ref : struct_ref->fields) { + if (auto why = detect_why_cant_serialize(field_ref->declared_type, is_pack)) { + return CantSerializeBecause("because field `" + struct_ref->name + "." + field_ref->name + "` of type `" + field_ref->declared_type->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (is_type_cellT(t_struct)) { + TypePtr cellT = struct_ref->substitutedTs->typeT_at(0); + if (auto why = detect_why_cant_serialize(cellT, is_pack)) { + return CantSerializeBecause("because type `" + cellT->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_union = any_type->try_as()) { + // a union can almost always be serialized if every of its variants can: + // - `T?` is TL/B `(Maybe T)` + // - `T1 | T2` is TL/B `(Either T1 T2)` (or, if opcodes manually set, just by opcodes) + // - `T1 | T2 | ...` is either by manual opcodes, or the compiler implicitly defines them + // so, even `int32 | int64 | int128` or `A | B | C | null` are serializable + // (unless corner cases occur, like duplicated opcodes, etc.) + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + if (variant == TypeDataNullLiteral::create()) { + continue; + } + if (auto why = detect_why_cant_serialize(variant, is_pack)) { + return CantSerializeBecause("because variant #" + std::to_string(i + 1) + " of type `" + variant->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (!t_union->or_null) { + std::string because_msg; + auto_generate_opcodes_for_union(t_union, because_msg); + if (!because_msg.empty()) { + return CantSerializeBecause("because could not automatically generate serialization prefixes for a union\n" + because_msg); + } + } + return {}; + } + + if (const auto* t_tensor = any_type->try_as()) { + for (int i = 0; i < t_tensor->size(); ++i) { + if (auto why = detect_why_cant_serialize(t_tensor->items[i], is_pack)) { + return CantSerializeBecause("because element `tensor." + std::to_string(i) + "` of type `" + t_tensor->items[i]->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { // it's built-in RemainingBitsAndRefs (slice) + return {}; + } + if (FunctionPtr f_pack = get_custom_pack_unpack_function(t_alias, true)) { + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (!check_declared_packToBuilder(t_alias, f_pack)) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` is declared incorrectly\nhint: it must accept 2 parameters and return nothing:\n> fun " + receiver_name + ".packToBuilder(self, mutate b: builder)"); + } + if (!f_pack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` can't be inlined; probably, it contains `return` in the middle"); + } + if (FunctionPtr f_unpack = get_custom_pack_unpack_function(t_alias, false)) { + if (!check_declared_unpackFromSlice(t_alias, f_unpack)) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` is declared incorrectly\nhint: it must accept 1 parameter and return an object:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + if (!f_unpack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle"); + } + } else if (!is_pack) { + return CantSerializeBecause("because type `" + receiver_name + "` defines a custom pack function, but does not define unpack\nhint: declare unpacker like this:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + return {}; + } + if (auto why = detect_why_cant_serialize(t_alias->underlying_type, is_pack)) { + return CantSerializeBecause("because alias `" + t_alias->as_human_readable() + "` expands to `" + t_alias->underlying_type->as_human_readable() + "`", why.value()); + } + return {}; + } + + // `builder` and `slice` can be used for writing, but not for reading + if (any_type == TypeDataBuilder::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); + } + if (any_type == TypeDataSlice::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `slice` can not be used for reading, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + } + + // serialization not available + // for common types, make a detailed explanation with a hint how to fix + + if (any_type == TypeDataInt::create()) { + return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); + } + if (any_type == TypeDataNullLiteral::create()) { + return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); + } + if (any_type == TypeDataTuple::create() || any_type->try_as()) { + return CantSerializeBecause("because tuples are not serializable\nhint: use tensors instead of tuples, they will work"); + } + + return CantSerializeBecause("because type `" + any_type->as_human_readable() + "` is not serializable"); + } +}; + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg) { + PackUnpackAvailabilityChecker checker; + if (auto why = checker.detect_why_cant_serialize(any_type, is_pack)) { + because_msg = why.value().because_msg; + return false; + } + return true; +} + +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); + } + return stack_offset; +} + + +// -------------------------------------------- +// high-level API for outer code +// + + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options) { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); + + tolk_assert(ir_options.size() == 1); // struct PackOptions + PackContext ctx(code, loc, rvect_builder, ir_options); + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); + + return rvect_cell; +} + +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options) { + PackContext ctx(code, loc, ir_builder, ir_options); // mutate this builder + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + return ir_builder; // return mutated builder +} + +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options) { + if (!mutate_slice) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); + ir_slice = std::move(slice_copy); + } + + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle + if (!mutate_slice && !estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } + return rvect_struct; +} + +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options) { + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); + + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + // if a struct has RemainingBitsAndRefs, don't test it for assertEnd + if (!estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } + return rvect_struct; +} + +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options) { + UnpackContext ctx(code, loc, ir_slice, ir_options); // mutate this slice + ctx.generate_skip_any(any_type); + + return ir_slice; // return mutated slice +} + +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { + StructPtr original_struct = load_info.original_struct; + StructPtr hidden_struct = load_info.hidden_struct; + tolk_assert(hidden_struct->fields.size() == load_info.ith_field_action.size()); + + const LazyStructLoadedState* loaded_state = lazy_variable->get_struct_state(original_struct); + tolk_assert(loaded_state && !loaded_state->was_loaded_once()); + loaded_state->mutate()->on_started_loading(hidden_struct); + + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.loadAndCheckOpcode(hidden_struct->opcode); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(!loaded_state->ith_field_was_loaded[field_idx]); + + // note that as opposed to regular loading, lazy loading doesn't return rvect, it fills stack slots (ir_obj) instead + switch (load_info.ith_field_action[field_idx]) { + case LazyStructLoadInfo::LoadField: { + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + tolk_assert(hidden_field->declared_type == original_field->declared_type); + std::vector ir_field = ctx.generate_unpack_any(hidden_field->declared_type); + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + code.emplace_back(loc, Op::_Let, std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width), std::move(ir_field)); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + } else { + tolk_assert(hidden_field->name == "(gap)"); + std::vector ir_gap = ctx.generate_unpack_any(hidden_field->declared_type); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_gap)); + } + break; + } + case LazyStructLoadInfo::SkipField: { + ctx.generate_skip_any(hidden_field->declared_type); + break; + } + case LazyStructLoadInfo::LazyMatchField: { + StructFieldPtr original_field = original_struct->find_field(hidden_field->name); + tolk_assert(original_field && hidden_field->declared_type == original_field->declared_type); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + break; + } + case LazyStructLoadInfo::SaveImmutableTail: { + std::vector ir_immutable_tail = code.create_tmp_var(TypeDataSlice::create(), loc, "(lazy-tail-slice)"); + code.emplace_back(loc, Op::_Let, ir_immutable_tail, lazy_variable->ir_slice); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_immutable_tail)); + break; + } + } + } + + // options.assertEndAfterReading is ignored by `lazy`, because tail fields may be skipped, it's okay +} + +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options) { + StructPtr original_struct = loaded_state->original_struct; + StructPtr hidden_struct = loaded_state->hidden_struct; + + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, lookup_function("beginCell")); + + PackContext ctx(code, loc, rvect_builder, ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.storeUint(code.create_int(loc, hidden_struct->opcode.pack_prefix, "(struct-prefix)"), hidden_struct->opcode.prefix_len); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(loaded_state->ith_field_was_loaded[field_idx]); + + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + std::vector ir_field(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field)); + } else { + std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field); + if (hidden_field->declared_type->unwrap_alias()->try_as()) { + ctx.storeSlice(ir_gap_or_tail[0]); + } else { + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail)); + } + if (hidden_field->name == "(tail)") { + break; + } + } + } + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), lookup_function("builder.endCell")); + + return rvect_cell; +} + +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { + tolk_assert(lazy_variable->ir_options.size() == 2); + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); + + return rvect_match; +} + +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj) { + tolk_assert(lazy_variable->ir_slice.size() == 1); + + // the call to `lazy_var.forceLoadLazyObject()` does not do anything: at the moment of analyzing, + // it had marked all the object as "used", all fields where loaded, and the slice points after the last field; + // so, just return the held slice + static_cast(code); + static_cast(loc); + static_cast(ir_obj); + + return lazy_variable->ir_slice; +} + +PackSize estimate_serialization_size(TypePtr any_type) { + EstimateContext ctx; + return ctx.estimate_any(any_type); +} + +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type) { + EstimateContext ctx; + PackSize pack_size = ctx.estimate_any(any_type); + + std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[1]}, td::make_refint(pack_size.max_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); + + FunctionPtr f_toTuple = lookup_function("T.__toTuple"); + std::vector ir_tuple = code.create_tmp_var(TypeDataTuple::create(), loc, "(result-tuple)"); + code.emplace_back(loc, Op::_Call, ir_tuple, ir_tensor, f_toTuple); + + return ir_tuple; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h new file mode 100644 index 000000000..7b4d02f66 --- /dev/null +++ b/tolk/pack-unpack-api.h @@ -0,0 +1,44 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "pack-unpack-serializers.h" +#include "tolk.h" + +namespace tolk { + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options); +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options); +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options); + +PackSize estimate_serialization_size(TypePtr any_type); +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); + +struct LazyStructLoadInfo; +struct LazyStructLoadedState; +struct LazyVariableLoadedState; + +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj); +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options); +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj); + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp new file mode 100644 index 000000000..5f1ebcc8f --- /dev/null +++ b/tolk/pack-unpack-serializers.cpp @@ -0,0 +1,1305 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-serializers.h" +#include "tolk.h" +#include "type-system.h" +#include "td/utils/crypto.h" + +/* + * This module implements serializing different types to/from cells. + * For any serializable TypePtr, we detect ISerializer, which can pack/unpack/skip/estimate size. + * See `get_serializer_for_type()`. + * Example: given an object of `struct A { f: int32 }` its type is TypeDataStruct(A), its serializer is + * "custom struct", which iterates fields, for field `f` its serializer is "intN" with N=32. + * + * Serializing compound types is complicated, involving transitioning IR variables. For example, to serialize + * `int8 | A` (it's Either), we have input rvect of size = 1 + width(A), generate dynamic IF ELSE, and in each branch, + * transition rvect slots to a narrowed type. Operating with transitions and runtime type checking are implemented + * in IR generation, here we just reference those prototypes. + * + * For high-level (de)serialization API, consider `pack-unpack-api.cpp`. + */ + +namespace tolk { + +class LValContext; +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); + +bool is_type_cellT(TypePtr any_type) { + if (const TypeDataStruct* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } + return false; +} + +// For any type alias, one can declare custom pack/unpack functions: +// > type TelegramString = slice +// > fun TelegramString.packToBuilder(self, mutate b: builder) { ... } +// > fun TelegramString.unpackFromSlice(mutate s: slice): TelegramString { ... } +// It's externally checked in advance that it's declared correctly. +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack) { + if (const TypeDataAlias* t_alias = receiver_type->try_as()) { + if (t_alias->alias_ref->is_instantiation_of_generic_alias()) { + // does not work for generic aliases currently, because `MyAlias.pack` was not instantiated earlier + return nullptr; + } + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (const Symbol* sym = lookup_global_symbol(receiver_name + (is_pack ? ".packToBuilder" : ".unpackFromSlice"))) { + return sym->try_as(); + } + } + return nullptr; +} + +// -------------------------------------------- +// options, context, common helpers +// +// some of the referenced functions are built-in, some are declared in stdlib +// serialization assumes that stdlib exists and is loaded correctly +// + + +PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options) + : code(code) + , loc(loc) + , f_storeInt(lookup_function("builder.storeInt")) + , f_storeUint(lookup_function("builder.storeUint")) + , ir_builder(std::move(ir_builder)) + , ir_builder0(this->ir_builder[0]) + , option_skipBitsNValidation(ir_options[0]) { +} + +void PackContext::storeInt(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeInt); +} + +void PackContext::storeUint(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + +void PackContext::storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const { + std::vector args = { ir_builder0, ir_idx, ir_len }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + +void PackContext::storeBool(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); +} + +void PackContext::storeCoins(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeCoins")); +} + +void PackContext::storeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeRef")); +} + +void PackContext::storeMaybeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); +} + +void PackContext::storeAddress(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); +} + +void PackContext::storeBuilder(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); +} + +void PackContext::storeSlice(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeSlice")); +} + +void PackContext::storeOpcode(PackOpcode opcode) const { + std::vector args = { ir_builder0, code.create_int(loc, opcode.pack_prefix, "(struct-prefix)"), code.create_int(loc, opcode.prefix_len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + + +UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options) + : code(code) + , loc(loc) + , f_loadInt(lookup_function("slice.loadInt")) + , f_loadUint(lookup_function("slice.loadUint")) + , f_skipBits(lookup_function("slice.skipBits")) + , ir_slice(std::move(ir_slice)) + , ir_slice0(this->ir_slice[0]) + , option_assertEndAfterReading(ir_options[0]) + , option_throwIfOpcodeDoesNotMatch(ir_options[1]) { +} + +std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadInt); + return result; +} + +std::vector UnpackContext::loadUint(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadUint); + return result; +} + +void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); +} + +void UnpackContext::skipBits(int len) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(skipW)") }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + +void UnpackContext::skipBits_var(var_idx_t ir_len) const { + std::vector args = { ir_slice0, ir_len }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + +void UnpackContext::assertEndIfOption() const { + Op& if_assertEnd = code.emplace_back(loc, Op::_If, std::vector{option_assertEndAfterReading}); + { + code.push_set_cur(if_assertEnd.block0); + Op& op_ends = code.emplace_back(loc, Op::_Call, std::vector{}, ir_slice, lookup_function("slice.assertEnd")); + op_ends.set_impure_flag(); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_assertEnd.block1); + code.close_pop_cur(loc); + } +} + +void UnpackContext::throwInvalidOpcode() const { + std::vector args_throw = { option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); +} + +const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr variant) const { + for (const MatchBlock& b : match_blocks) { + if (b.arm_variant->get_type_id() == variant->get_type_id()) { + return &b; + } + } + tolk_assert(false); +} + +void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const { + if (!is_statement) { + // if it's `match` expression (not statement), then every arm has a result, assigned to a whole `match` result + ir_arm_result = transition_to_target_type(std::move(ir_arm_result), code, arm_block->block_expr_type, match_expr_type, loc); + code.emplace_back(loc, Op::_Let, ir_match_expr_result, std::move(ir_arm_result)); + } else if (add_return_to_all_arms) { + // if it's `match` statement, even if an arm is an expression, it's void, actually + // moreover, if it's the last statement in a function, add implicit "return" to all match cases to produce IFJMP + code.emplace_back(loc, Op::_Return); + } +} + + +// -------------------------------------------- +// serializers with pack/unpack/skip/estimate +// +// for every struct field, for every atomic type, a corresponding (de)serialization instruction is generated +// we generate IR code (Ops), not ASM directly; so, all later IR analysis will later take place +// some of them are straightforward, e.g., call a predefined function for intN and coins +// some are complicated, e.g., for Either we should check a union type at runtime while packing, +// and while unpacking, read a prefix, follow different branches, and construct a resulting union +// + + +struct ISerializer { + virtual ~ISerializer() = default; + + virtual void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) = 0; + virtual std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + + virtual void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + virtual PackSize estimate(const EstimateContext* ctx) = 0; +}; + +struct S_IntN final : ISerializer { + const int n_bits; + const bool is_unsigned; + + explicit S_IntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (is_unsigned) { + ctx->storeUint(rvect[0], n_bits); + } else { + ctx->storeInt(rvect[0], n_bits); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (is_unsigned) { + return ctx->loadUint(n_bits, "(loaded-uint)"); + } else { + return ctx->loadInt(n_bits, "(loaded-int)"); + } + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_VariadicIntN final : ISerializer { + const int n_bits; // only 16 and 32 available + const bool is_unsigned; + + explicit S_VariadicIntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_storeVarInt = lookup_function("builder.__storeVarInt"); + std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadVarInt = lookup_function("slice.__loadVarInt"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-varint)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt); + return result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + if (n_bits == 32) { + return PackSize(5, 253); + } else { + return PackSize(4, 124); // same as `coins` + } + } +}; + +struct S_BitsN final : ISerializer { + const int n_bits; + + explicit S_BitsN(int n_width, bool is_bits) + : n_bits(is_bits ? n_width : n_width * 8) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + + Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNValidation}); + { + code.push_set_cur(if_disabled_by_user.block0); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_disabled_by_user.block1); + FunctionPtr f_assert = lookup_function("__throw_if_unless"); + constexpr int EXCNO = 9; + + std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); + code.emplace_back(loc, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); + std::vector args_assert0 = { code.create_int(loc, EXCNO, "(excno)"), ir_counts[1], code.create_int(loc, 1, "") }; + Op& op_assert0 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); + op_assert0.set_impure_flag(); + std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-n)"); + code.emplace_back(loc, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(n-bits)")}, lookup_function("_==_")); + std::vector args_assertN = { code.create_int(loc, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(loc, 0, "") }; + Op& op_assertN = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); + op_assertN.set_impure_flag(); + code.close_pop_cur(loc); + } + + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadBits = lookup_function("slice.loadBits"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(loadW)") }; + std::vector ir_result = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-slice)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadBits); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_Bool final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeBool(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return ctx->loadInt(1, "(loaded-bool)"); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(1); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1); + } +}; + +struct S_RawTVMcell final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector dummy_loaded = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 1, 1); + } +}; + +struct S_RawTVMcellOrNull final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeMaybeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadMaybeRef = lookup_function("slice.loadMaybeRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadMaybeRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_skipMaybeRef = lookup_function("slice.skipMaybeRef"); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ctx->ir_slice, f_skipMaybeRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1, 1, 0, 1); + } +}; + +struct S_Coins final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeCoins(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadCoins); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(4, 124); + } +}; + +struct S_Address final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeAddress(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-addr)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + return ir_address; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // we can't do just + // ctx->skipBits(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + // we can't do just + // return PackSize(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; but since addr_extern is very-very uncommon, don't consider it + return PackSize(2, 2 + 1 + 8 + 256); + } +}; + +struct S_RemainingBitsAndRefs final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_rem_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(remainder)"); + code.emplace_back(loc, Op::_Let, ir_rem_slice, ctx->ir_slice); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, std::vector{}, lookup_function("createEmptySlice")); + return ir_rem_slice; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(tmp-builder)"); + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(tmp-cell)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + code.emplace_back(loc, Op::_Call, ir_cell, ir_builder, f_endCell); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ir_cell, f_beginParse); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + +struct S_Builder final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + std::vector args = { ctx->ir_builder0, rvect[0] }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeBuilder")); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + +struct S_Slice final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + +struct S_Null final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + // while `null` itself is not serializable, it may be contained inside a union: + // `int32 | int64 | null`, for example; + // then the compiler generates prefixes for every variant, and `null` variant does nothing + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null)"); + code.emplace_back(loc, Op::_Call, ir_null, std::vector{}, lookup_function("__null")); + return ir_null; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Never final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.empty()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return {}; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Maybe final : ISerializer { + const TypeDataUnion* t_union; + TypePtr or_null; + + explicit S_Maybe(const TypeDataUnion* t_union) + : t_union(t_union) + , or_null(t_union->or_null) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_null = pre_compile_is_type(code, t_union, TypeDataNullLiteral::create(), rvect, loc, "(is-null)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_null); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 0, "(maybeBit)"), 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 1, "(maybeBit)"), 1); + rvect = transition_to_target_type(std::move(rvect), code, t_union, t_union->or_null, loc); + ctx->generate_pack_any(t_union->or_null, std::move(rvect)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-maybe)"); + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_maybe = ctx->generate_unpack_any(t_union->or_null); + rvect_maybe = transition_to_target_type(std::move(rvect_maybe), code, t_union->or_null, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_maybe)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(maybe-null)"); + code.emplace_back(loc, Op::_Call, rvect_null, std::vector{}, lookup_function("__null")); + rvect_null = transition_to_target_type(std::move(rvect_null), code, TypeDataNullLiteral::create(), t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_null)); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_union->or_null); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize maybe_size = ctx->estimate_any(t_union->or_null); + return PackSize(1, 1 + maybe_size.max_bits, 0, maybe_size.max_refs); + } +}; + +struct S_Either final : ISerializer { + const TypeDataUnion* t_union; + TypePtr t_left; + TypePtr t_right; + + explicit S_Either(const TypeDataUnion* t_union) + : t_union(t_union) + , t_left(t_union->variants[0]) + , t_right(t_union->variants[1]) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_right = pre_compile_is_type(code, t_union, t_right, rvect, loc, "(is-right)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_right); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 1, "(eitherBit)"), 1); + std::vector rvect_right = transition_to_target_type(std::vector(rvect), code, t_union, t_right, loc); + ctx->generate_pack_any(t_right, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 0, "(eitherBit)"), 1); + std::vector rvect_left = transition_to_target_type(std::move(rvect), code, t_union, t_left, loc); + ctx->generate_pack_any(t_left, std::move(rvect_left)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-either)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_right = ctx->generate_unpack_any(t_right); + rvect_right = transition_to_target_type(std::move(rvect_right), code, t_right, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_left = ctx->generate_unpack_any(t_left); + rvect_left = transition_to_target_type(std::move(rvect_left), code, t_left, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_left)); + code.close_pop_cur(loc); + } + return ir_result; + } + + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + for (const LazyMatchOptions::MatchBlock& m : options.match_blocks) { + if (m.arm_variant == nullptr) { // `else => ...` not allowed for Either + // it's not the best place to fire an error, but let it be + throw ParseError(loc, "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); + } + } + tolk_assert(options.match_blocks.size() == 2); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_right); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_left); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_right); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->generate_skip_any(t_left); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize either_size = EstimateContext::minmax(ctx->estimate_any(t_left), ctx->estimate_any(t_right)); + return EstimateContext::sum(PackSize(1), either_size); + } +}; + +struct S_MultipleConstructors final : ISerializer { + const TypeDataUnion* t_union; + std::vector opcodes; + + explicit S_MultipleConstructors(const TypeDataUnion* t_union, std::vector&& opcodes) + : t_union(t_union) + , opcodes(std::move(opcodes)) { + tolk_assert(this->opcodes.size() == t_union->variants.size()); + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + for (int i = 0; i < t_union->size() - 1; ++i) { + TypePtr variant = t_union->variants[i]; + std::vector ir_eq_ith = pre_compile_is_type(code, t_union, variant, rvect, loc, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_eq_ith)); + code.push_set_cur(if_op.block0); + std::vector ith_rvect = transition_to_target_type(std::vector(rvect), code, t_union, variant, loc); + ctx->storeUint(code.create_int(loc, opcodes[i].pack_prefix, "(ith-prefix)"), opcodes[i].prefix_len); + ctx->generate_pack_any(variant, std::move(ith_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + // we're inside the last ELSE + TypePtr last_variant = t_union->variants.back(); + std::vector last_rvect = transition_to_target_type(std::move(rvect), code, t_union, last_variant, loc); + ctx->storeUint(code.create_int(loc, opcodes.back().pack_prefix, "(ith-prefix)"), opcodes.back().prefix_len); + ctx->generate_pack_any(last_variant, std::move(last_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + for (int i = 0; i < t_union->size() - 1; ++i) { + code.close_pop_cur(loc); // close all outer IFs + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // assume that opcodes (either automatically generated or manually specified) + // form a valid prefix tree, and the order of reading does not matter; we'll definitely match the one; + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-union)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + ctx->throwInvalidOpcode(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + return ir_result; + } + + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector opcodes_order_mapping(t_union->size(), -1); + const LazyMatchOptions::MatchBlock* else_block = nullptr; + for (int i = 0; i < static_cast(options.match_blocks.size()); ++i) { + if (options.match_blocks[i].arm_variant) { + int variant_idx = t_union->get_variant_idx(options.match_blocks[i].arm_variant); + tolk_assert(variant_idx != -1); + opcodes_order_mapping[i] = variant_idx; + } else { + tolk_assert(else_block == nullptr); + else_block = &options.match_blocks[i]; + } + } + + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + StructData::PackOpcode opcode = opcodes[opcodes_order_mapping[i]]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + std::vector ith_result = pre_compile_expr(options.match_blocks[i].v_body, code); + options.save_match_result_on_arm_end(code, loc, &options.match_blocks[i], std::move(ith_result), ir_result); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + ctx->throwInvalidOpcode(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize variants_size = ctx->estimate_any(t_union->variants[0], PrefixEstimateMode::DoNothingAlreadyIncluded); + PackSize prefix_size(opcodes[0].prefix_len); + + for (int i = 1; i < t_union->size(); ++i) { + variants_size = EstimateContext::minmax(variants_size, ctx->estimate_any(t_union->variants[i], PrefixEstimateMode::DoNothingAlreadyIncluded)); + prefix_size = EstimateContext::minmax(prefix_size, PackSize(opcodes[i].prefix_len)); + } + + return EstimateContext::sum(variants_size, prefix_size); + } +}; + +struct S_Tensor final : ISerializer { + const TypeDataTensor* t_tensor; + + explicit S_Tensor(const TypeDataTensor* t_tensor) + : t_tensor(t_tensor) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + int stack_offset = 0; + for (TypePtr item : t_tensor->items) { + int stack_width = item->get_width_on_stack(); + std::vector item_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(item, std::move(item_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == t_tensor->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector tensor_vars; + tensor_vars.reserve(t_tensor->get_width_on_stack()); + for (TypePtr item : t_tensor->items) { + std::vector item_vars = ctx->generate_unpack_any(item); + tensor_vars.insert(tensor_vars.end(), item_vars.begin(), item_vars.end()); + } + tolk_assert(static_cast(tensor_vars.size()) == t_tensor->get_width_on_stack()); + return tensor_vars; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + for (TypePtr item : t_tensor->items) { + ctx->generate_skip_any(item); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum = PackSize(0); + for (TypePtr item : t_tensor->items) { + sum = EstimateContext::sum(sum, ctx->estimate_any(item)); + } + return sum; + } +}; + +struct S_CustomStruct final : ISerializer { + StructPtr struct_ref; + + explicit S_CustomStruct(StructPtr struct_ref) + : struct_ref(struct_ref) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixWriteMode::WritePrefixOfStruct) { + ctx->storeOpcode(struct_ref->opcode); + } + + int stack_offset = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(field_ref->declared_type, std::move(field_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == TypeDataStruct::create(struct_ref)->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + int total_stack_w = TypeDataStruct::create(struct_ref)->get_width_on_stack(); + std::vector ir_struct; + ir_struct.reserve(total_stack_w); + for (StructFieldPtr field_ref : struct_ref->fields) { + std::vector field_vars = ctx->generate_unpack_any(field_ref->declared_type); + ir_struct.insert(ir_struct.end(), field_vars.begin(), field_vars.end()); + } + tolk_assert(static_cast(ir_struct.size()) == total_stack_w); + return ir_struct; + } + + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + const LazyMatchOptions::MatchBlock* when_block = nullptr; // Point => ... + const LazyMatchOptions::MatchBlock* else_block = nullptr; // else => ... + for (const LazyMatchOptions::MatchBlock& match_block : options.match_blocks) { + if (match_block.arm_variant) { + tolk_assert(match_block.arm_variant->equal_to(TypeDataStruct::create(struct_ref))); + when_block = &match_block; + } else { + else_block = &match_block; + } + } + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + StructData::PackOpcode opcode = struct_ref->opcode; + if (opcode.exists()) { // it's `match` over a struct (makes sense for a struct with prefix and `else` branch) + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + } else { + code.emplace_back(loc, Op::_Let, ir_prefix_eq, std::vector{code.create_int(loc, -1, "(true)")}); + } + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + { + code.push_set_cur(if_op.block0); + std::vector when_result = pre_compile_expr(when_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, when_block, std::move(when_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } + code.close_pop_cur(loc); + } + + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + ctx->generate_skip_any(field_ref->declared_type); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum(0); + + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixEstimateMode::IncludePrefixOfStruct) { + sum = EstimateContext::sum(sum, PackSize(struct_ref->opcode.prefix_len)); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + sum = EstimateContext::sum(sum, ctx->estimate_any(field_ref->declared_type)); + } + return sum; + } +}; + +struct S_CustomReceiverForPackUnpack final : ISerializer { + TypePtr receiver_type; + + explicit S_CustomReceiverForPackUnpack(TypePtr receiver_type) + : receiver_type(receiver_type) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_pack = get_custom_pack_unpack_function(receiver_type, true); + tolk_assert(f_pack && f_pack->does_accept_self() && f_pack->inferred_return_type->get_width_on_stack() == 0); + std::vector vars_per_arg = { std::move(rvect), ctx->ir_builder }; + std::vector ir_mutated_builder = gen_inline_fun_call_in_place(code, TypeDataBuilder::create(), loc, f_pack, nullptr, false, vars_per_arg); + code.emplace_back(loc, Op::_Let, ctx->ir_builder, std::move(ir_mutated_builder)); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_unpack = get_custom_pack_unpack_function(receiver_type, false); + tolk_assert(f_unpack && f_unpack->inferred_return_type->get_width_on_stack() == receiver_type->get_width_on_stack()); + TypePtr ret_type = TypeDataTensor::create({TypeDataSlice::create(), receiver_type}); + std::vector ir_slice_and_res = gen_inline_fun_call_in_place(code, ret_type, loc, f_unpack, nullptr, false, {ctx->ir_slice}); + code.emplace_back(loc, Op::_Let, ctx->ir_slice, std::vector{ir_slice_and_res.front()}); + return std::vector(ir_slice_and_res.begin() + 1, ir_slice_and_res.end()); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // just load and ignore the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + + +// -------------------------------------------- +// automatically generate opcodes +// +// for union types like `T1 | T2 | ...`, if prefixes for structs are not manually specified, +// the compiler generates a valid prefix tree: for `int32 | int64 | int128` it's '00' '01' '10'; +// it works both for structs (with unspecified prefixes) and primitives: `int32 | A | B` is ok; +// but if some prefixes are specified, some not — it's an error +// + + +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg) { + const TypeDataUnion* t_union = union_type->try_as(); + std::vector result; + result.reserve(t_union->size()); + + int n_have_opcode = 0; + bool has_null = false; + StructPtr last_struct_with_opcode = nullptr; // for error message + StructPtr last_struct_no_opcode = nullptr; + for (TypePtr variant : t_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + if (variant_struct->struct_ref->opcode.exists()) { + n_have_opcode++; + last_struct_with_opcode = variant_struct->struct_ref; + } else { + last_struct_no_opcode = variant_struct->struct_ref; + } + } else if (variant == TypeDataNullLiteral::create()) { + has_null = true; + } + } + + // `A | B | C`, all of them have opcodes — just use them; + // for instance, `A | B` is not either (0/1 + data), but uses manual opcodes + if (n_have_opcode == t_union->size()) { + for (TypePtr variant : t_union->variants) { + result.push_back(variant->unwrap_alias()->try_as()->struct_ref->opcode); + } + return result; + } + + // invalid: `A | B | C`, some of them have opcodes, some not; + // example: `A | B` if A has opcode, B not; + // example: `int32 | A` if A has opcode; + // example: `int32 | int64 | A` if A has opcode; + if (n_have_opcode) { + if (last_struct_with_opcode && last_struct_no_opcode) { + because_msg = "because struct `" + last_struct_with_opcode->as_human_readable() + "` has opcode, but `" + last_struct_no_opcode->as_human_readable() + "` does not\nhint: manually specify opcodes to all structures"; + } else { + because_msg = "because of mixing primitives and struct `" + last_struct_with_opcode->as_human_readable() + "` with serialization prefix\nhint: extract primitives to single-field structs and provide prefixes"; + } + return result; + } + + // okay, none of the opcodes are specified, generate a prefix tree; + // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; + // if `null` exists, it's 0, all others are 1+tree: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D; + // if no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C + int n_without_null = t_union->size() - has_null; + int prefix_len = static_cast(std::ceil(std::log2(n_without_null))); + int cur_prefix = 0; + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataNullLiteral::create()) { + result.emplace_back(0, 1); + } else if (has_null) { + result.emplace_back((1< get_serializer_for_type(TypePtr any_type) { + if (const auto* t_intN = any_type->try_as()) { + if (t_intN->is_variadic) { + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } + if (const auto* t_bitsN = any_type->try_as()) { + return std::make_unique(t_bitsN->n_width, t_bitsN->is_bits); + } + if (any_type == TypeDataCoins::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBool::create()) { + return std::make_unique(); + } + if (any_type == TypeDataCell::create() || is_type_cellT(any_type)) { + return std::make_unique(); + } + if (any_type == TypeDataAddress::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBuilder::create()) { + return std::make_unique(); + } + if (any_type == TypeDataSlice::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNullLiteral::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNever::create()) { + return std::make_unique(); + } + + if (const auto* t_struct = any_type->try_as()) { + return std::make_unique(t_struct->struct_ref); + } + + if (const auto* t_union = any_type->try_as()) { + // `T?` is always `(Maybe T)`, even if T has custom opcode (opcode will follow bit '1') + if (t_union->or_null) { + TypePtr or_null = t_union->or_null->unwrap_alias(); + if (or_null == TypeDataCell::create() || is_type_cellT(or_null)) { + return std::make_unique(); + } + return std::make_unique(t_union); + } + + // `T1 | T2` is `(Either T1 T2)` (0/1 + contents) unless they both have custom prefixes + bool all_have_opcode = true; + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + all_have_opcode &= variant_struct && variant_struct->struct_ref->opcode.exists(); + } + if (t_union->size() == 2 && !all_have_opcode) { + return std::make_unique(t_union); + } + // `T1 | T2 | T3`, probably nullable, probably with primitives, probably with custom opcodes; + // compiler is able to generate serialization prefixes automatically; + // and this type is valid, it was checked earlier + std::string err_msg; + std::vector opcodes = auto_generate_opcodes_for_union(t_union, err_msg); + tolk_assert(err_msg.empty()); + return std::make_unique(t_union, std::move(opcodes)); + } + + if (const auto* t_tensor = any_type->try_as()) { + return std::make_unique(t_tensor); + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { + return std::make_unique(); + } + if (get_custom_pack_unpack_function(t_alias, true)) { + return std::make_unique(t_alias); + } + return get_serializer_for_type(t_alias->underlying_type); + } + + // this should not be reachable, serialization availability is checked earlier + throw Fatal("type `" + any_type->as_human_readable() + "` can not be serialized"); +} + + +void PackContext::generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode) const { + PrefixWriteMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->pack(this, code, loc, std::move(rvect)); + this->prefix_mode = backup; +} + +std::vector UnpackContext::generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + std::vector result = get_serializer_for_type(any_type)->unpack(this, code, loc); + this->prefix_mode = backup; + return result; +} + +void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->skip(this, code, loc); + this->prefix_mode = backup; +} + +std::vector UnpackContext::generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const { + std::unique_ptr serializer = get_serializer_for_type(any_type); + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + tolk_assert(false); +} + +PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { + PrefixEstimateMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + PackSize result = get_serializer_for_type(any_type)->estimate(this); + this->prefix_mode = backup; + return result; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h new file mode 100644 index 000000000..ab11733aa --- /dev/null +++ b/tolk/pack-unpack-serializers.h @@ -0,0 +1,169 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +using PackOpcode = StructData::PackOpcode; + +struct PackSize { + int min_bits; + int max_bits; + int min_refs; + int max_refs; + + bool is_unpredictable_infinity() const { + return max_bits >= 9999; + } + + explicit PackSize(int exact_bits) + : min_bits(exact_bits), max_bits(exact_bits), min_refs(0), max_refs(0) { + } + PackSize(int min_bits, int max_bits) + : min_bits(min_bits), max_bits(max_bits), min_refs(0), max_refs() { + } + PackSize(int min_bits, int max_bits, int min_refs, int max_refs) + : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { + } + + static PackSize unpredictable_infinity() { + return PackSize(0, 9999, 0, 4); + } +}; + + +enum class PrefixWriteMode { + WritePrefixOfStruct, + DoNothingAlreadyWritten, +}; + +class PackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_storeInt; + const FunctionPtr f_storeUint; + mutable PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct; + +public: + const std::vector ir_builder; + const var_idx_t ir_builder0; + const var_idx_t option_skipBitsNValidation; + + PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); + + PrefixWriteMode get_prefix_mode() const { return prefix_mode; } + + void storeInt(var_idx_t ir_idx, int len) const; + void storeUint(var_idx_t ir_idx, int len) const; + void storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const; + void storeBool(var_idx_t ir_idx) const; + void storeCoins(var_idx_t ir_idx) const; + void storeRef(var_idx_t ir_idx) const; + void storeMaybeRef(var_idx_t ir_idx) const; + void storeAddress(var_idx_t ir_idx) const; + void storeBuilder(var_idx_t ir_idx) const; + void storeSlice(var_idx_t ir_idx) const; + void storeOpcode(PackOpcode opcode) const; + + void generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct) const; +}; + + +enum class PrefixReadMode { + LoadAndCheck, + DoNothingAlreadyLoaded, +}; + +struct LazyMatchOptions { + struct MatchBlock { + TypePtr arm_variant; // left of `V => ...`; nullptr for `else => ...` + AnyExprV v_body; // right of `V => ...` + TypePtr block_expr_type; // for match expression, if `V => expr`, it's expr's inferred_type + }; + + TypePtr match_expr_type; // type of `match` expression, `void` for statement + bool is_statement; // it's `match` statement, not expression, so it does not return any result + bool add_return_to_all_arms; // it's the last statement in a function, add "return" to its cases for better Fift code + std::vector match_blocks; + + const MatchBlock* find_match_block(TypePtr variant) const; + void save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const; +}; + +class UnpackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_loadInt; + const FunctionPtr f_loadUint; + const FunctionPtr f_skipBits; + mutable PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck; + +public: + const std::vector ir_slice; + const var_idx_t ir_slice0; + const var_idx_t option_assertEndAfterReading; + const var_idx_t option_throwIfOpcodeDoesNotMatch; + + UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options); + + PrefixReadMode get_prefix_mode() const { return prefix_mode; } + + std::vector loadInt(int len, const char* debug_desc) const; + std::vector loadUint(int len, const char* debug_desc) const; + void loadAndCheckOpcode(PackOpcode opcode) const; + void skipBits(int len) const; + void skipBits_var(var_idx_t ir_len) const; + void assertEndIfOption() const; + void throwInvalidOpcode() const; + + std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + std::vector generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const; +}; + + +enum class PrefixEstimateMode { + IncludePrefixOfStruct, + DoNothingAlreadyIncluded, +}; + +class EstimateContext { + mutable PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct; + +public: + + PrefixEstimateMode get_prefix_mode() const { return prefix_mode; } + + static PackSize minmax(PackSize a, PackSize b) { + return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); + } + static PackSize sum(PackSize a, PackSize b) { + return PackSize(a.min_bits + b.min_bits, std::min(9999, a.max_bits + b.max_bits), a.min_refs + b.min_refs, a.max_refs + b.max_refs); + } + + PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; +}; + + +bool is_type_cellT(TypePtr any_type); +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1561aa403..f22b0ab44 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -17,11 +17,16 @@ #include "tolk.h" #include "src-file.h" #include "ast.h" +#include "ast-aux-data.h" #include "ast-visitor.h" #include "type-system.h" #include "common/refint.h" -#include "constant-evaluator.h" -#include +#include "smart-casts-cfg.h" +#include "pack-unpack-api.h" +#include "gen-entrypoints.h" +#include "generics-helpers.h" +#include "send-message-api.h" +#include "gen-entrypoints.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -41,7 +46,7 @@ * Example: `nullableInt!`; for `nullableInt` inferred_type is `int?`, and target_type is `int` * (this doesn't lead to stack reorganization, but in case `nullableTensor!` does) * (inferred_type of `nullableInt!` is `int`, and its target_type depends on its usage). - * The same mechanism will work for union types in the future. + * Example: `var a: int|slice = 5`. This `5` should be extended as "5 1" (5 for value, 1 for type_id of `int`). */ namespace tolk { @@ -51,6 +56,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); void process_any_statement(AnyV v, CodeBlob& code); +static AnyV stmt_before_immediate_return = nullptr; // The goal of VarsModificationWatcher is to detect such cases: `return (x, x += y, x)`. // Without any changes, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - incorrect @@ -99,6 +105,23 @@ class VarsModificationWatcher { static VarsModificationWatcher vars_modification_watcher; +static int calc_offset_on_stack(const TypeDataTensor* t_tensor, int index_at) { + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + return stack_offset; +} + +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); + } + return stack_offset; +} + + // Main goal of LValContext is to handle non-primitive lvalues. At IR level, a usual local variable // exists, but on its change, something non-trivial should happen. // Example: `globalVar = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `SetGlob "globVar" = $6` @@ -142,20 +165,25 @@ class LValContext { // example: `global v: (int, int); v.1 = 5`, implicit var is created `$tmp = 5`, and when it's modified, // we need to partially update w; essentially, apply_partially_rewrite() above will be called struct ModifiedFieldOfGlobal { - AnyExprV tensor_obj; - int index_at; + AnyExprV tensor_obj; // it's a tensor or struct + int index_at; // for tensors, it's index_at; for structs, it's field_idx std::vector lval_ir_idx; void apply(CodeBlob& code, SrcLocation loc) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); - const TypeDataTensor* t_tensor = tensor_obj->inferred_type->try_as(); - tolk_assert(t_tensor); - int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); + + int stack_width, stack_offset; + TypePtr obj_type = tensor_obj->inferred_type->unwrap_alias(); + if (const TypeDataTensor* t_tensor = obj_type->try_as()) { + stack_width = t_tensor->items[index_at]->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_tensor, index_at); + } else if (const TypeDataStruct* t_struct = obj_type->try_as()) { + stack_width = t_struct->struct_ref->get_field(index_at)->declared_type->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_struct->struct_ref, index_at); + } else { + tolk_assert(false); } std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); @@ -182,7 +210,7 @@ class LValContext { code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); - FunctionPtr builtin_sym = lookup_global_symbol("tupleSetAt")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.set"); code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); local_lval.after_let(std::move(tuple_ir_idx), code, loc); } @@ -255,6 +283,110 @@ class LValContext { } }; +// the purpose of this class is having a call `f(a1,a2,...)` when f has asm arg_order, to check +// whether it's safe to rearrange arguments (to evaluate them in arg_order right here for fewer stack manipulations) +// or it's unsafe, and we should evaluate them left-to-right; +// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe; +// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` / `f(s.loadInt(), s.loadInt())` is unsafe; +// the same rules are used to check an object literal: is it safe to convert `{y:expr, x:expr}` to declaration order {x,y} +class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { + bool has_side_effects = false; + +protected: + void visit(V v) override { + has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure() || v->fun_maybe->has_mutate_params(); + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static bool is_safe_to_reorder(V v) { + for (const LocalVarData& param : v->fun_maybe->parameters) { + if (param.declared_type->get_width_on_stack() != 1) { + return false; + } + } + + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_args(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_arg(i)->get_expr()); + } + if (v->dot_obj_is_self) { + visitor.ASTVisitorFunctionBody::visit(v->get_self_obj()); + } + return !visitor.has_side_effects; + } + + static bool is_safe_to_reorder(V v) { + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_fields(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_field(i)->get_init_val()); + } + return !visitor.has_side_effects; + } +}; + +// when a call to `f()` was inlined, f's body was processed, leaving some state +// that should be cleared upon next inlining; +// for instance, ir_idx of local variables point to caller (where f was inlined) +class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { + void visit(V v) override { + if (!v->marked_as_redef) { + v->var_ref->mutate()->assign_ir_idx({}); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + tolk_assert(fun_ref->is_inlined_in_place()); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + fun_ref->get_param(i).mutate()->assign_ir_idx({}); + } + + parent::visit(v_function->get_body()); + } +}; + + +// CodeBlob has a mapping [st => ptr] +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(LocalVarPtr var_ref) const { + for (const LazyVarRefAtCodegen& stored : lazy_variables) { + if (stored.var_ref == var_ref) { + return stored.var_state; + } + } + return nullptr; +} + +// detect `st` by vertex "st" +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return get_lazy_variable(var_ref); + } + } + return nullptr; +} + + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -269,12 +401,35 @@ static AnyExprV unwrap_not_null_operator(AnyExprV v) { static V calc_sink_leftmost_obj(V v) { AnyExprV leftmost_obj = unwrap_not_null_operator(v->get_obj()); while (auto v_dot = leftmost_obj->try_as()) { - if (!v_dot->is_target_indexed_access()) { + if (!v_dot->is_target_indexed_access() && !v_dot->is_target_struct_field()) { break; } leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); } - return leftmost_obj->type == ast_reference ? leftmost_obj->as() : nullptr; + return leftmost_obj->kind == ast_reference ? leftmost_obj->as() : nullptr; +} + +// ternary `x ? y : z` can be optimized to asm `CONDSEL` (not IF/ELSE), if y and z don't require evaluation; +// example when can: `cond ? 2 : null`, `x == null ? some_var : obj.field`; +// example when not: `cond ? f() : g()` and other non-trivial arguments +static bool is_ternary_arg_trivial_for_condsel(AnyExprV v, bool require_1slot = true) { + if (require_1slot && v->inferred_type->get_width_on_stack() != 1) { + return false; + } + if (v->kind == ast_int_const || v->kind == ast_string_const || v->kind == ast_bool_const || + v->kind == ast_null_keyword || v->kind == ast_reference) { + return true; + } + if (auto v_par = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_par->get_expr(), require_1slot); + } + if (auto v_dot = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_dot->get_obj(), false); + } + if (auto v_cast = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_cast->get_expr(), require_1slot); + } + return false; } @@ -349,13 +504,15 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co } static std::vector pre_compile_tensor(CodeBlob& code, const std::vector& args, - LValContext* lval_ctx = nullptr) { - std::vector types_list; - types_list.reserve(args.size()); - for (AnyExprV item : args) { - types_list.push_back(item->inferred_type); + LValContext* lval_ctx = nullptr, const TypeDataTensor* tensor_target_type = nullptr) { + if (tensor_target_type == nullptr) { + std::vector types_list; + types_list.reserve(args.size()); + for (AnyExprV item : args) { + types_list.push_back(item->inferred_type); + } + tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); } - const TypeDataTensor* tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); std::vector> res_lists = pre_compile_tensor_inner(code, args, tensor_target_type, lval_ctx); std::vector res; for (const std::vector& list : res_lists) { @@ -366,12 +523,12 @@ static std::vector pre_compile_tensor(CodeBlob& code, const std::vect static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" - if (lhs->type == ast_typed_tuple && rhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple && rhs->kind == ast_bracket_tuple) { // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); - std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); code.emplace_back(loc, Op::_Let, left, rvect); local_lval.after_let(std::move(left), code, loc); std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); @@ -379,12 +536,12 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // [lhs] = rhs; it's un-tuple to N left vars - if (lhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple) { LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, nullptr); - const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->try_as(); + const TypeDataBrackets* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); @@ -393,7 +550,7 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually - if (lhs->type == ast_local_var_lhs || (lhs->type == ast_reference && lhs->as()->sym->try_as())) { + if (lhs->kind == ast_local_var_lhs || (lhs->kind == ast_reference && lhs->as()->sym->try_as())) { std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); @@ -410,16 +567,209 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { + FunctionPtr eq_sym = lookup_function("_==_"); + FunctionPtr isnull_sym = lookup_function("__isNull"); + FunctionPtr not_sym = lookup_function("!b_"); + std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); + + const TypeDataUnion* lhs_union = expr_type->try_as(); + if (!lhs_union) { + // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 + bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(types_eq ? -1 : 0)); + } else if (lhs_union->is_primitive_nullable() && cmp_type == TypeDataNullLiteral::create()) { + // `int?` is `null` for primitive 1-slot nullables, they hold either value of TVM NULL, no extra union tag slot + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + } else if (lhs_union->is_primitive_nullable()) { + // `int?` is `int` (check for null actually) / `int?` is `builder` (compile-time false actually) + bool cant_happen = lhs_union->or_null->get_type_id() != cmp_type->get_type_id(); + if (cant_happen) { + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(0)); + } else { + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + code.emplace_back(loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); + } + } else { + // `int | slice` is `int`, check type id + std::vector typeid_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(type-id)"); + code.emplace_back(loc, Op::_IntConst, typeid_ir_idx, td::make_refint(cmp_type->get_type_id())); + code.emplace_back(loc, Op::_Call, result_ir_idx, std::vector{typeid_ir_idx[0], expr_ir_idx.back()}, eq_sym); + } + + return result_ir_idx; +} + static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, - std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { + std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc, + bool arg_order_already_equals_asm = false) { std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); if (!fun_ref->is_marked_as_pure()) { op.set_impure_flag(); } + if (arg_order_already_equals_asm) { + op.set_arg_order_already_equals_asm_flag(); + } return rvect; } +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, V v_call, const std::vector>& vars_per_arg) { + SrcLocation loc = v_call->loc; + FunctionPtr called_f = v_call->fun_maybe; + + if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + + const LazyVariableLoadedState* lazy_variable = v_call->dot_obj_is_self ? code.get_lazy_variable(v_call->get_self_obj()) : nullptr; + + if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { + // in: object Lazy (partially loaded), out: Cell + std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx + return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); + } + if (f_name == "T.toCell") { + // in: object T, out: Cell (just a cell, wrapped) + std::vector ir_obj = vars_per_arg[0]; + return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj), vars_per_arg[1]); + } + if (f_name == "T.fromCell") { + // in: cell, out: object T + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); + } + if (f_name == "T.fromSlice") { + // in: slice, out: object T, input slice NOT mutated + std::vector ir_slice = vars_per_arg[0]; + return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false, vars_per_arg[1]); + } + if (f_name == "Cell.load") { + // in: cell, out: object T + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); + } + if (f_name == "slice.loadAny") { + // in: slice, out: object T, input slice is mutated, so prepend self before an object + var_idx_t ir_self = vars_per_arg[0][0]; + std::vector ir_slice = vars_per_arg[0]; + std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), true, vars_per_arg[1]); + std::vector ir_result = {ir_self}; + ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); + return ir_result; + } + if (f_name == "slice.skipAny") { + // in: slice, out: the same slice, with a shifted pointer + std::vector ir_slice = vars_per_arg[0]; + return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice), vars_per_arg[1]); + } + if (f_name == "builder.storeAny") { + // in: builder and object T, out: mutated builder + std::vector ir_builder = vars_per_arg[0]; + std::vector ir_obj = vars_per_arg[1]; + return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::move(ir_obj), vars_per_arg[2]); + } + if (f_name == "T.estimatePackSize") { + return generate_estimate_size_call(code, loc, typeT); + } + if (f_name == "T.forceLoadLazyObject") { + // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) + if (!lazy_variable) { + fire(code.fun_ref, v_call->loc, "this method is applicable to lazy variables only"); + } + std::vector ir_obj = vars_per_arg[0]; + return generate_lazy_object_finish_loading(code, loc, lazy_variable, std::move(ir_obj)); + } + } + + if (called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + + if (f_name == "createMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + if (f_name == "createExternalLogMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createExternalLogMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + } + + if (called_f->name == "address.buildSameAddressInAnotherShard") { + std::vector ir_self_address = vars_per_arg[0]; + std::vector ir_shard_options = vars_per_arg[1]; + return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); + } + if (called_f->name == "AutoDeployAddress.buildAddress") { + std::vector ir_self = vars_per_arg[0]; + return generate_AutoDeployAddress_buildAddress(code, loc, std::move(ir_self)); + } + if (called_f->name == "AutoDeployAddress.addressMatches") { + std::vector ir_self = vars_per_arg[0]; + std::vector ir_address = vars_per_arg[1]; + return generate_AutoDeployAddress_addressMatches(code, loc, std::move(ir_self), std::move(ir_address)); + } + + tolk_assert(false); +} + +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { + tolk_assert(vars_per_arg.size() == f_inlined->parameters.size()); + for (int i = 0; i < f_inlined->get_num_params(); ++i) { + const LocalVarData& param_i = f_inlined->get_param(i); + if (!param_i.is_used_as_lval() && !param_i.is_mutate_parameter()) { + // if param used for reading only, pass the same ir_idx as for an argument + // it decreases number of tmp variables and leads to better optimizations + // (being honest, it's quite strange that copy+LET may lead to more stack permutations) + param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); + } else { + std::vector ith_param = code.create_var(param_i.declared_type, loc, param_i.name); + code.emplace_back(loc, Op::_Let, ith_param, vars_per_arg[i]); + param_i.mutate()->assign_ir_idx(std::move(ith_param)); + } + } + + std::vector rvect_call = code.create_tmp_var(ret_type, loc, "(inlined-return)"); + std::vector* backup_outer_inline = code.inline_rvect_out; + FunctionPtr backup_cur_fun = code.fun_ref; + bool backup_inline_before_return = code.inlining_before_immediate_return; + auto backup_lazy_variables = code.lazy_variables; + code.inline_rvect_out = &rvect_call; + code.inlining_before_immediate_return = is_before_immediate_return; + code.fun_ref = f_inlined; + // specially handle `point.getX()` if point is a lazy var: to make `self.toCell()` work and `self.x` asserted; + // (only methods preserve lazy, `getXOf(point)` does not, though theoretically can be done) + const LazyVariableLoadedState* lazy_receiver = self_obj ? code.get_lazy_variable(self_obj) : nullptr; + if (lazy_receiver) { + LocalVarPtr self_var_ref = &f_inlined->parameters[0]; // `self` becomes lazy while inlining + code.lazy_variables.emplace_back(self_var_ref, lazy_receiver); // (points to the same slice, immutable tail, etc.) + } + + auto v_ast_root = f_inlined->ast_root->as(); + auto v_block = v_ast_root->get_body()->as(); + process_any_statement(v_block, code); + + if (f_inlined->has_mutate_params() && f_inlined->inferred_return_type == TypeDataVoid::create()) { + std::vector mutated_vars; + for (const LocalVarData& p_sym: f_inlined->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + code.emplace_back(loc, Op::_Let, rvect_call, std::move(mutated_vars)); + } + + ClearStateAfterInlineInPlace visitor; + visitor.start_visiting_function(f_inlined, v_ast_root); + + code.fun_ref = backup_cur_fun; + code.inline_rvect_out = backup_outer_inline; + code.inlining_before_immediate_return = backup_inline_before_return; + code.lazy_variables = std::move(backup_lazy_variables); + return rvect_call; +} + // "Transition to target (runtime) type" is the following process. // Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. // `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). @@ -428,10 +778,17 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL // `null` (inferred_type) is 1 stack slots, but target_type is 3, we should add 2 nulls. // Another example: `var t1 = (1, null); var t2: (int, (int,int)?) = t1;`. // Then t1's rvect is 2 vars (1 and null), but t1's `null` should be converted to 3 stack slots (resulting in 4 total). -// The same mechanism will work for union types in the future. -// Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. +// The same mechanism works for union types, but there is a union tag (UTag) slot instead of null flag. +// Another example: `var i: int|slice = 5;`. This "5" is represented as "5 1" (5 for value, 1 is type_id of `int`). GNU_ATTRIBUTE_NOINLINE static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(rvect.size()) == original_type->get_width_on_stack()); +#endif + + original_type = original_type->unwrap_alias(); + target_type = target_type->unwrap_alias(); + // pass `T` to `T` // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components if (target_type == original_type) { @@ -439,117 +796,304 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorget_width_on_stack(); - const TypeDataNullable* t_nullable = target_type->try_as(); - const TypeDataNullable* o_nullable = original_type->try_as(); + int orig_w = original_type->get_width_on_stack(); // = rvect.size() + const TypeDataUnion* t_union = target_type->try_as(); + const TypeDataUnion* o_union = original_type->try_as(); + + // most common case, simple nullability: + // - `int` to `int?` + // - `null` to `int?` + // - `int8?` to `int16?` + // - `null` to `StructWith1Int?` + // in general, pass `T1` to `T2?` when `T2?` still occupies 1 stack slot (value or TVM NULL) + if (t_union && t_union->is_primitive_nullable() && orig_w == 1) { + // rvect has 1 slot, either value or TVM NULL + return rvect; + } - // handle `never` - // it may occur due to smart cast and in unreachable branches - // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted - if (original_type == TypeDataNever::create()) { - std::vector dummy_rvect; - dummy_rvect.reserve(target_w); - for (int i = 0; i < target_w; ++i) { - dummy_rvect.push_back(code.create_tmp_var(TypeDataUnknown::create(), loc, "(never)")[0]); - } - return dummy_rvect; + // smart cast of a primitive 1-slot nullable: + // - `int?` to `int` + // - `int?` to `null` + // - `StructWith1Int?` to `null` + // this value (one slot) is either a TVM primitive or TVM NULL at runtime + if (o_union && o_union->is_primitive_nullable() && target_w == 1) { + // rvect has 1 slot, but its contents is compile-time guaranteed to match target_type + return rvect; } - if (target_type == TypeDataNever::create()) { - return {}; + + // pass `T` to `never` + // it occurs due to smart cast, in unreachable branches, for example `if (intVal == null) { return intVal; }` + // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted + if (target_type == TypeDataNever::create() || original_type == TypeDataNever::create() || target_type == TypeDataUnknown::create()) { + return rvect; } - // pass `null` to `T?` - // for primitives like `int?`, no changes in rvect, null occupies the same TVM slot - // for tensors like `(int,int)?`, `null` is represented as N nulls + 1 null flag, insert N nulls - if (t_nullable && original_type == TypeDataNullLiteral::create()) { - tolk_assert(rvect.size() == 1); - if (target_w == 1 && !t_nullable->is_primitive_nullable()) { // `null` to `()?` - rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - code.emplace_back(loc, Op::_IntConst, rvect, td::make_refint(0)); - } - if (target_w > 1) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect.reserve(target_w + 1); - for (int i = 1; i < target_w - 1; ++i) { - std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, builtin_sym); - rvect.push_back(ith_null[0]); - } - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(0)); - rvect.push_back(null_flag_ir_idx); + // smart cast to a primitive 1-slot nullable: + // - `int | slice | null` to `slice?` + // - `A | int | null` to `int?` + // so, originally a type occupies N slots, but needs to be converted to 1 slot + if (t_union && t_union->is_primitive_nullable() && orig_w > 0) { + // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` + tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); + // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack + // `T1 | T2 | ... | null` occupies N+1 slots, where the last is for UTag + // when it holds null value, N slots are null, and UTag slot is 0 (it's type_id of TypeDataNullLiteral) + return {rvect[rvect.size() - 2]}; + } + + // pass `null` to `T?` when T is wide (stores some nulls and UTag=0 at runtime) + // - `null` to `(int, int)?` + // - `null` to `int | slice | null` + // to represent a non-primitive null value, we need N nulls + 1 null flag (UTag=0, type_id of TypeDataNullLiteral) + if (t_union && target_w > 1 && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->has_null()); + FunctionPtr null_sym = lookup_function("__null"); + rvect.reserve(target_w); // keep rvect[0], it's already null + for (int i = 1; i < target_w - 1; ++i) { + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + rvect.push_back(ith_null[0]); } + std::vector last_null = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_null, td::make_refint(0)); + rvect.push_back(last_null[0]); return rvect; } - // pass `T` to `T?` - // for primitives like `int?`, no changes in rvect: `int` and `int?` occupy the same TVM slot (null is represented as NULL TVM value) - // for passing `(int, int)` to `(int, int)?` / `(int, null)` to `(int, (int,int)?)?`, add a null flag equals to 0 - if (t_nullable && !o_nullable) { - if (!t_nullable->is_primitive_nullable()) { - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_nullable->inner, loc); - tolk_assert(target_w == static_cast(rvect.size() + 1)); - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(-1)); - rvect.push_back(null_flag_ir_idx); + + // pass `null` to nullable empty type + // - `null` to `()?` + // - `null` to `EmptyStruct?` + // - `null` to `Empty1 | Empty2 | null` + // so, rvect contains TVM NULL, but instead, we should push UTag=0 + if (t_union && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->has_null() && target_w == 1); + std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); + return new_rvect; + } + + // smart cast of a wide nullable union to plain `null` + // - `(int, int)?` to `null` + // - `int | slice | null` to `null` + if (o_union && target_type == TypeDataNullLiteral::create() && orig_w > 1) { + tolk_assert(o_union->has_null()); + // if we are here, it's guaranteed that original value holds null + // it means, that its shape is N nulls + 1 UTag (equals 0) + return {rvect[rvect.size() - 2]}; + } + + // smart cast of nullable empty tensor to plain `null` + // - `()?` to `null` + // - `EmptyStruct?` to `null` + // - `Empty1 | Empty2 | null` to `null` + // so, rvect contains UTag, we need TVM NULL + if (o_union && target_type == TypeDataNullLiteral::create()) { + tolk_assert(orig_w == 1 && o_union->has_null()); + FunctionPtr null_sym = lookup_function("__null"); + std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); + return new_rvect; + } + + // pass primitive 1-slot `T?` to a wider nullable union + // - `int?` to `int | slice | null` + // - `slice?` to `(int, int) | slice | builder | null` + // so, originally `T?` is 1-slot, but needs to be converted to N+1 slots, keeping its value + if (o_union && o_union->is_primitive_nullable() && t_union) { + tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); + // the transformation is tricky: + // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") + // when value is not null, we need to get "... value {type_id}" + // this can be done only via IFs at runtime; luckily, this case is very uncommon in practice + // for "...", we might need N-1 nulls: `int?` to `(int,int,int) | int | null` is `(null) (null) value/(null) 0/1` + FunctionPtr null_sym = lookup_function("__null"); + std::vector new_rvect; + new_rvect.resize(target_w); + for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + new_rvect[i] = ith_null[0]; } + new_rvect[target_w - 2] = rvect[0]; // value + new_rvect[target_w - 1] = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)")[0]; + + std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); + FunctionPtr isnull_sym = lookup_function("__isNull"); + code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); + Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); + code.push_set_cur(if_op.block0); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(0)); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(o_union->or_null->get_type_id())); + code.close_pop_cur(loc); + return new_rvect; + } + + // extend a single type into a union type + // - `int` to `int | slice` + // - `int` to `int | (int, int) | null` + // - `(int, int)` to `(int, int, cell) | builder | (int, int)` + // - `(int, null)` to `(int, (int, int)?) | ...`: mind transition + // - `(int, null)` to `(int, int | slice | null) | ...`: mind transition + // so, probably need to prepend some nulls, and need to append UTag + if (t_union && !o_union) { + TypePtr t_subtype = t_union->calculate_exact_variant_to_fit_rhs(original_type); + tolk_assert(t_subtype && target_w > t_subtype->get_width_on_stack()); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_subtype, loc); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); + for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { + FunctionPtr null_sym = lookup_function("__null"); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); + } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); + + std::vector last_utag = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_utag, td::make_refint(t_subtype->get_type_id())); + rvect.push_back(last_utag[0]); return rvect; } - // pass `T1?` to `T2?` - // for example, `int8?` to `int16?` - // transition inner types, leaving nullable flag unchanged for tensors - if (t_nullable && o_nullable) { - if (target_w > 1) { - var_idx_t null_flag_ir_idx = rvect.back(); - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_nullable->inner, t_nullable->inner, loc); - rvect.push_back(null_flag_ir_idx); - } + + // smart cast a union type to a single type + // - `int | slice` to `int` + // - `int | (int, int) | null` to `int` + // - `(int, (int, int)?) | ...` to `(int, null)`: mind transition + // so, cut off UTag and probably some unused tags from the start + if (o_union && !t_union) { + TypePtr o_subtype = o_union->calculate_exact_variant_to_fit_rhs(target_type); + tolk_assert(o_subtype && orig_w > o_subtype->get_width_on_stack()); + rvect = std::vector(rvect.begin() + orig_w - o_subtype->get_width_on_stack() - 1, rvect.end() - 1); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_subtype, target_type, loc); return rvect; } - // pass `T?` to `null` - // it may occur due to smart cast, when a `T?` variable is guaranteed to be always null - // (for instance, always-null `(int,int)?` will be represented as 1 TVM NULL value, not 3) - if (target_type == TypeDataNullLiteral::create() && original_type->can_rhs_be_assigned(target_type)) { - tolk_assert(o_nullable || original_type == TypeDataUnknown::create()); - if (o_nullable && !o_nullable->is_primitive_nullable()) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, rvect, std::vector{}, builtin_sym); + + // extend a union type to a wider one + // - `int | slice` to `int | slice | builder` + // - `int | slice` to `int | (int, int) | slice | null` + // so, both original and target have UTag slot, but rvect probably needs to be prepended by nulls + if (t_union && o_union && t_union->size() >= o_union->size()) { + tolk_assert(target_w >= orig_w && t_union->has_all_variants_of(o_union)); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - orig_w); + for (int i = 0; i < target_w - orig_w; ++i) { + FunctionPtr null_sym = lookup_function("__null"); + std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null_ir_idx[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); return rvect; } - // pass `T?` to `T` (or, more generally, `T1?` to `T2`) - // it may occur due to operator `!` or smart cast - // for primitives like `int?`, no changes in rvect - // for passing `(int, int)?` to `(int, int)`, drop the null flag from the tail - // for complex scenarios like passing `(int, (int,int)?)?` to `(int, null)`, recurse the call - // (it may occur on `someF(t = (3,null))` when `(3,null)` at first targeted to lhs, but actually its result is rhs) - if (!t_nullable && o_nullable) { - if (!o_nullable->is_primitive_nullable()) { - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type->try_as()->inner, target_type, loc); - } + + // smart cast a wider union type to a narrow one + // - `int | slice | builder` to `int | slice` + // - `int | (int, int) | slice | null` to `int | slice` + // so, both original and target have UTag slot, but rvect needs to be cut off from the left + if (t_union && o_union) { + tolk_assert(target_w <= orig_w && o_union->has_all_variants_of(t_union)); + rvect = std::vector(rvect.begin() + (orig_w - target_w), rvect.end()); return rvect; } + // pass `bool` to `int` // in code, it's done via `as` operator, like `boolVar as int` // no changes in rvect, boolVar is guaranteed to be -1 or 0 at TVM level - if (target_type == TypeDataInt::create() && original_type == TypeDataBool::create()) { + if (original_type == TypeDataBool::create() && target_type == TypeDataInt::create()) { + return rvect; + } + + // pass `bool` to `int8` + // same as above + if (original_type == TypeDataBool::create() && target_type->try_as()) { + return rvect; + } + + // pass `int8` to `int` + // it comes from auto cast when an integer (even a literal) is assigned to intN + // to changes in rvect, intN is int at TVM level + if (target_type == TypeDataInt::create() && original_type->try_as()) { + return rvect; + } + + // pass `coins` to `int` + // same as above + if (target_type == TypeDataInt::create() && original_type == TypeDataCoins::create()) { + return rvect; + } + + // pass `int` to `int8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type == TypeDataInt::create() && target_type->try_as()) { + return rvect; + } + + // pass `int` to `coins` + // same as above + if (original_type == TypeDataInt::create() && target_type == TypeDataCoins::create()) { + return rvect; + } + + // pass `int8` to `int16` / `int8` to `uint8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } + + // pass `int8` to `coins` + // same as above + if (target_type == TypeDataCoins::create() && original_type->try_as()) { + return rvect; + } + + // pass `coins` to `int8` + // same as above + if (original_type == TypeDataCoins::create() && target_type->try_as()) { + return rvect; + } + + // pass `bytes32` to `slice` + // in code, it's probably done with `as` operator + // no changes in rvect, since bitsN is slice at TVM level + if (target_type == TypeDataSlice::create() && original_type->try_as()) { + return rvect; + } + + // pass `slice` to `bytes32` + // same as above + if (original_type == TypeDataSlice::create() && target_type->try_as()) { + return rvect; + } + + // pass `bytes32` to `bytes64` / `bits128` to `bytes16` + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } + + // pass a typed tuple `[int, int]` to an untyped (via `as` operator) + if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect if (target_type == TypeDataUnknown::create()) { return rvect; } + // pass `unknown` to something // probably, it comes from `arg` in exception, it's inferred as `unknown` and could be cast to any value if (original_type == TypeDataUnknown::create()) { tolk_assert(rvect.size() == 1); return rvect; } + // pass tensor to tensor, e.g. `(1, null)` to `(int, slice?)` / `(1, null)` to `(int, (int,int)?)` // every element of rhs tensor should be transitioned if (target_type->try_as() && original_type->try_as()) { @@ -569,10 +1113,48 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && original_type->try_as()) { - tolk_assert(target_type->get_width_on_stack() == original_type->get_width_on_stack()); + if (target_type->try_as() && original_type->try_as()) { + tolk_assert(target_w == 1 && orig_w == 1); + return rvect; + } + + // pass callable to callable + // their types aren't exactly equal, but they match (containing aliases, for example) + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(rvect.size() == 1); + return rvect; + } + + // pass struct A to struct B + // different structs are typically not assignable, but Wrapper> is ok to Wrapper> + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(target_type->can_rhs_be_assigned(original_type) && orig_w == target_w); + return rvect; + } + + // pass slice to address + // no changes in rvect: address is TVM slice under the hood + if (original_type == TypeDataSlice::create() && target_type == TypeDataAddress::create()) { + return rvect; + } + + // pass address to slice + // same, no changes in rvect + if (original_type == TypeDataAddress::create() && target_type == TypeDataSlice::create()) { + return rvect; + } + + // pass `Cell` to `cell`, e.g. `setContractData(obj.toCell())` + if (target_type == TypeDataCell::create() && original_type->try_as()) { + tolk_assert(orig_w == 1 && original_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); + return rvect; + } + // and vice versa, `cell as Cell` + if (original_type == TypeDataCell::create() && target_type->try_as()) { + tolk_assert(target_w == 1 && target_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); return rvect; } @@ -595,7 +1177,7 @@ static std::vector transition_to_target_type(std::vector&& #ifndef TOLK_DEBUG GNU_ATTRIBUTE_ALWAYS_INLINE #endif -static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { if (target_type != original_type) { rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); } @@ -621,15 +1203,10 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co return local_ir_idx; } if (GlobalConstPtr const_ref = sym->try_as()) { - if (const_ref->is_int_const()) { - std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_IntConst, rvect, const_ref->as_int_const()); - return rvect; - } else { - std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->as_slice_const()); - return rvect; - } + tolk_assert(lval_ctx == nullptr); + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); + auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + return pre_compile_expr(v_force_loc, code, const_ref->declared_type); } if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); @@ -708,7 +1285,7 @@ static std::vector process_binary_operator(V v, v_1->mutate()->assign_inferred_type(TypeDataInt::create()); auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); - v_b_ne_0->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_b_ne_0->mutate()->assign_fun_ref(lookup_function("_!=_")); std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); @@ -721,8 +1298,22 @@ static std::vector process_binary_operator(V v, code.close_pop_cur(v->loc); return transition_to_target_type(std::move(rvect), code, target_type, v); } + if (t == tok_eq || t == tok_neq) { + if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + FunctionPtr f_sliceEq = lookup_function("slice.bitsEqual"); + std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); + std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); + std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); + code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); + if (t == tok_neq) { + FunctionPtr not_sym = lookup_function("!b_"); + code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); + } + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + } - throw UnexpectedASTNodeType(v, "process_binary_operator"); + throw UnexpectedASTNodeKind(v, "process_binary_operator"); } static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { @@ -735,11 +1326,16 @@ static std::vector process_ternary_operator(V v std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); - + if (v->get_cond()->is_always_true) { code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + } else if (v->inferred_type->get_width_on_stack() == 1 && is_ternary_arg_trivial_for_condsel(v->get_when_true()) && is_ternary_arg_trivial_for_condsel(v->get_when_false())) { + std::vector ir_true = pre_compile_expr(v->get_when_true(), code, v->inferred_type); + std::vector ir_false = pre_compile_expr(v->get_when_false(), code, v->inferred_type); + std::vector condsel_args = { cond[0], ir_true[0], ir_false[0] }; + code.emplace_back(v->loc, Op::_Call, rvect, std::move(condsel_args), lookup_function("__condsel")); } else { Op& if_op = code.emplace_back(v->loc, Op::_If, cond); code.push_set_cur(if_op.block0); @@ -754,56 +1350,206 @@ static std::vector process_ternary_operator(V v } static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->cast_to_type; + TypePtr child_target_type = v->type_node->resolved_type; std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->get_expr()->inferred_type; - if (const auto* as_nullable = child_target_type->try_as()) { - child_target_type = as_nullable->inner; +static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr cmp_type = v->type_node->resolved_type->unwrap_alias(); + bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` + tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error + + std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); + + if (v->is_negated) { + FunctionPtr not_sym = lookup_function("!b_"); + code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); } + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); +} + +static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); + TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { - std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); - std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); - TypePtr expr_type = v->get_expr()->inferred_type; - - if (const TypeDataNullable* t_nullable = expr_type->try_as()) { - if (!t_nullable->is_primitive_nullable()) { - std::vector zero_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->loc, "(zero)"); - code.emplace_back(v->loc, Op::_IntConst, zero_ir_idx, td::make_refint(0)); - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{expr_ir_idx.back(), zero_ir_idx[0]}, eq_sym); +static std::vector process_lazy_operator(V v, CodeBlob& code, TypePtr target_type) { + // `lazy Storage.fromSlice(s)` does not load anything here, it only saves a slice for future loads; + // "future loads" are special auxiliary AST vertices "load x" that were inserted in pipe-lazy-load-insertions.cpp + auto v_call = v->get_expr()->try_as(); + tolk_assert(v_call && v_call->fun_maybe); + + FunctionPtr called_f = v_call->fun_maybe; + if (called_f->is_code_function()) { // `lazy loadStorage()` is allowed, it contains just `return ...`, inline it here + auto f_body = called_f->ast_root->as()->get_body()->as(); + tolk_assert(f_body->size() == 1 && f_body->get_item(0)->kind == ast_return_statement); + auto f_returns = f_body->get_item(0)->as(); + v_call = f_returns->get_return_value()->try_as(); + tolk_assert(v_call && v_call->fun_maybe && v_call->fun_maybe->is_builtin_function()); + called_f = v_call->fun_maybe; + } + + // only predefined built-in functions are allowed for lazy loading + tolk_assert(called_f->is_builtin_function() && called_f->is_instantiation_of_generic_function()); + std::string_view f_name = called_f->base_fun_ref->name; + std::vector ir_slice = code.create_var(TypeDataSlice::create(), v->loc, "lazyS"); + bool has_passed_options = false; + if (f_name == "T.fromSlice") { + std::vector passed_slice = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Let, ir_slice, std::move(passed_slice)); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "T.fromCell") { + std::vector ir_cell = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "Cell.load") { + std::vector ir_cell = pre_compile_expr(v_call->get_callee()->try_as()->get_obj(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 1; + } else { + tolk_assert(false); + } + + // on `var p = lazy Point.fromSlice(s, options)`, save s and options (lazy_variable) + AnyExprV v_options = has_passed_options ? v_call->get_arg(v_call->get_num_args() - 1)->get_expr() : called_f->parameters.back().default_value; + std::vector ir_options = pre_compile_expr(v_options, code, called_f->parameters[1].declared_type); + const LazyVariableLoadedState* lazy_variable = new LazyVariableLoadedState(v->dest_var_ref->declared_type, std::move(ir_slice), std::move(ir_options)); + code.lazy_variables.emplace_back(v->dest_var_ref, lazy_variable); + + // initially, all contents of `p` is filled by nulls, but before `p.x` or any other field usages, + // they will be loaded by separate AST aux vertices; + // same for unions: `val msg = lazy MyMsgUnion`, msg is N+1 nulls, but next lazy `match` will transition slots, + // which will be filled by loads + std::vector ir_null = gen_op_call(code, TypeDataNullLiteral::create(), v->loc, {}, lookup_function("__null"), "(init-null)"); + std::vector ir_initial_nulls(v->dest_var_ref->ir_idx.size(), ir_null[0]); + return transition_to_target_type(std::move(ir_initial_nulls), code, target_type, v); +} + +static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + + int n_arms = v->get_arms_count(); + std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); + std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); + + if (!n_arms) { // `match (subject) {}` + tolk_assert(v->is_statement()); + return {}; + } + + // it's either `match` by type (all arms patterns are types) or `match` by expression + bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + bool last_else_branch = v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + // detect whether `match` is exhaustive + bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier + || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier + || last_else_branch; + + // `else` is not allowed in `match` by type; this was not fired at type checking, + // because it might turned out to be a lazy match, where `else` is allowed; + // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) + if (is_match_by_type && last_else_branch) { + throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + } + + // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 2 (exhaustive): `match (v) { -1 => ... 0 => ... else => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` + // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } + FunctionPtr eq_sym = lookup_function("_==_"); + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + auto v_ith_arm = v->get_arm(i); + std::vector eq_ith_ir_idx; + if (is_match_by_type) { + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type->unwrap_alias(); + tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error + eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__isNull")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, expr_ir_idx, builtin_sym); + std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); + tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); + eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm->loc, "(arm-cond-eq)"); + code.emplace_back(v_ith_arm->loc, Op::_Call, eq_ith_ir_idx, std::vector{subj_ir_idx[0], ith_ir_idx[0]}, eq_sym); } - } else { - bool always_null = expr_type == TypeDataNullLiteral::create(); - code.emplace_back(v->loc, Op::_IntConst, isnull_ir_idx, td::make_refint(always_null ? -1 : 0)); + Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op.block0); + if (v->is_statement()) { + pre_compile_expr(v_ith_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_ith_arm->loc, Op::_Return); + } + } else { + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); // open ELSE } - if (v->is_negated) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{isnull_ir_idx}, not_sym); + if (is_exhaustive) { + // we're inside the last ELSE + auto v_last_arm = v->get_arm(n_arms - 1); + if (v->is_statement()) { + pre_compile_expr(v_last_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_last_arm->loc, Op::_Return); + } + } else { + std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + } + // close all outer IFs + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + code.close_pop_cur(v->loc); } - return transition_to_target_type(std::move(isnull_ir_idx), code, target_type, v); + + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); } static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) if (!v->is_target_fun_ref()) { - TypePtr obj_type = v->get_obj()->inferred_type; - int index_at = std::get(v->target); + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); + // `user.id`; internally, a struct (an object) is a tensor + if (const auto* t_struct = obj_type->try_as()) { + StructFieldPtr field_ref = std::get(v->target); + // handle `globalObj.field = rhs`, special case, then the global will be read on demand + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-field)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), field_ref->field_idx, lval_ir_idx); + return lval_ir_idx; + } + } + // handle `lazyPoint.x`, assert that slot for "x" is loaded (ensure lazy-loading correctness); + // same for `val msg = lazy MyMsgUnion; match(...) msg.field` inside a specific variant (struct_ref) + if (const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(v->get_obj())) { + lazy_variable->assert_field_loaded(t_struct->struct_ref, field_ref); + } + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + int stack_width = field_ref->declared_type->get_width_on_stack(); + int stack_offset = calc_offset_on_stack(t_struct->struct_ref, field_ref->field_idx); + std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + // an object field might be smart cast at this point, for example we're in `if (user.t != null)` + // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations + // (from original rvect = (vars of user.t) to fit smart cast) + rvect = transition_to_target_type(std::move(rvect), code, field_ref->declared_type, v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } // `tensorVar.0` if (const auto* t_tensor = obj_type->try_as()) { - // handle `tensorVar.0 = rhs` if tensors is a global, special case, then the global will be read on demand + int index_at = std::get(v->target); + // handle `globalTensorVar.0 = rhs`, special case, then the global will be read on demand if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); @@ -814,10 +1560,7 @@ static std::vector process_dot_access(V v, CodeBlob& // since a tensor of N elems are N vars on a stack actually, calculate offset std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); - } + int stack_offset = calc_offset_on_stack(t_tensor, index_at); std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations @@ -826,20 +1569,21 @@ static std::vector process_dot_access(V v, CodeBlob& return transition_to_target_type(std::move(rvect), code, target_type, v); } // `tupleVar.0` - if (obj_type->try_as() || obj_type->try_as()) { + if (obj_type->try_as() || obj_type->try_as()) { + int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); return lval_ir_idx; } - // `tupleVar.0` as rvalue: the same as "tupleAt(tupleVar, 0)" written in terms of IR vars + // `tupleVar.0` as rvalue: the same as "tuple.get(tupleVar, 0)" written in terms of IR vars std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values - FunctionPtr builtin_sym = lookup_global_symbol("tupleAt")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.get"); code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); @@ -871,7 +1615,7 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } - std::vector params_types = v->get_callee()->inferred_type->try_as()->params_types; + std::vector params_types = v->get_callee()->inferred_type->unwrap_alias()->try_as()->params_types; const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); std::vector args_vars; @@ -887,29 +1631,59 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect), code, target_type, v); } - int delta_self = v->is_dot_call(); - AnyExprV obj_leftmost = nullptr; + // fill args for evaluation: dot object + passed arguments + parameters defaults if not all passed + AnyExprV obj_leftmost = v->get_self_obj(); + int delta_self = obj_leftmost != nullptr; std::vector args; - args.reserve(delta_self + v->get_num_args()); + args.reserve(fun_ref->get_num_params()); if (delta_self) { - args.push_back(v->get_dot_obj()); - obj_leftmost = v->get_dot_obj(); - while (obj_leftmost->type == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { - obj_leftmost = obj_leftmost->as()->get_dot_obj(); + args.push_back(obj_leftmost); + while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->get_self_obj() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + obj_leftmost = obj_leftmost->as()->get_self_obj(); } } for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + for (int i = delta_self + v->get_num_args(); i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + tolk_assert(param_ref->has_default_value()); + SrcLocation last_loc = args.empty() ? v->loc : args.back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(last_loc, param_ref->default_value, aux_data, param_ref->declared_type); + args.push_back(v_force_loc); + } + // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self std::vector params_types = fun_ref->inferred_full_type->try_as()->params_types; + + // if fun_ref has asm arg_order, maybe it's safe to swap arguments here (to put them onto a stack in the right way); + // (if it's not safe, arguments are evaluated left-to-right, involving stack transformations later) + bool arg_order_already_equals_asm = false; + int asm_self_idx = 0; + if (!fun_ref->arg_order.empty() && CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v)) { + std::vector new_args(args.size()); + std::vector new_params_types(params_types.size()); + for (int i = 0; i < static_cast(fun_ref->arg_order.size()); ++i) { + int real_i = fun_ref->arg_order[i]; + new_args[i] = args[real_i]; + new_params_types[i] = params_types[real_i]; + if (real_i == 0) { + asm_self_idx = i; + } + } + args = std::move(new_args); + params_types = std::move(new_params_types); + arg_order_already_equals_asm = true; + } + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); TypePtr op_call_type = v->inferred_type; TypePtr real_ret_type = v->inferred_type; - if (delta_self && fun_ref->does_return_self()) { + if (obj_leftmost && fun_ref->does_return_self()) { real_ret_type = TypeDataVoid::create(); if (!fun_ref->parameters[0].is_mutate_parameter()) { op_call_type = TypeDataVoid::create(); @@ -930,40 +1704,63 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)"); + std::vector rvect_call; + if (fun_ref->is_compile_time_special_gen()) { + rvect_call = gen_compile_time_code_instead_of_fun_call(code, v, vars_per_arg); + } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v->loc, v->fun_maybe, v->get_self_obj(), v == stmt_before_immediate_return, vars_per_arg); + } else { + rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + } if (fun_ref->has_mutate_params()) { LValContext local_lval; std::vector left; for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + int real_i = arg_order_already_equals_asm ? i == 0 && delta_self ? asm_self_idx : fun_ref->arg_order[i - delta_self] : i; if (fun_ref->parameters[i].is_mutate_parameter()) { - AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[i]; + AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[real_i]; tolk_assert(arg_i->is_lvalue || i == 0); if (arg_i->is_lvalue) { std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); } else { - left.insert(left.end(), vars_per_arg[0].begin(), vars_per_arg[0].end()); + left.insert(left.end(), vars_per_arg[asm_self_idx].begin(), vars_per_arg[asm_self_idx].end()); } } } std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); left.insert(left.end(), rvect.begin(), rvect.end()); vars_modification_watcher.trigger_callbacks(left, v->loc); - code.emplace_back(v->loc, Op::_Let, left, rvect_apply); + code.emplace_back(v->loc, Op::_Let, left, rvect_call); local_lval.after_let(std::move(left), code, v->loc); - rvect_apply = rvect; + rvect_call = rvect; } if (obj_leftmost && fun_ref->does_return_self()) { if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain - rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); + rvect_call = pre_compile_expr(obj_leftmost, code, nullptr); } else { // temporary object, not lvalue, pre_compile_expr - rvect_apply = vars_per_arg[0]; + rvect_call = vars_per_arg[asm_self_idx]; } } - return transition_to_target_type(std::move(rvect_apply), code, target_type, v); + return transition_to_target_type(std::move(rvect_call), code, target_type, v); +} + +static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex "braced expression" (currently, only `match` arms) + std::vector implicit_rvect; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_rvect.empty()); + implicit_rvect = pre_compile_expr(v_return->get_expr(), code); + } else { + process_any_statement(item, code); + } + } + return transition_to_target_type(std::move(implicit_rvect), code, target_type, v); } static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -973,7 +1770,7 @@ static std::vector process_tensor(V v, CodeBlob& code, Ty return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { +static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work v->error("[...] can not be used as lvalue here"); } @@ -983,31 +1780,132 @@ static std::vector process_typed_tuple(V v, CodeBlob return transition_to_target_type(std::move(left), code, target_type, v); } +static std::vector process_object_literal_shuffled(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // creating an object like `Point { y: getY(), x: getX() }`, where fields order doesn't match declaration; + // as opposed to a non-shuffled version `{x:..., y:...}`, we should at first evaluate fields as they created, + // and then to place them in a correct order + std::vector tensor_items; // create a tensor of literal fields values + std::vector target_types; + tensor_items.reserve(v->get_body()->get_num_fields()); + target_types.reserve(v->get_body()->get_num_fields()); + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + StructFieldPtr field_ref = v->struct_ref->find_field(v_field->get_field_name()); + tensor_items.push_back(v_field->get_init_val()); + target_types.push_back(field_ref->declared_type); + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector literal_rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + + std::vector rvect = code.create_tmp_var(TypeDataStruct::create(v->struct_ref), v->loc, "(object)"); + int stack_offset = 0; + for (StructFieldPtr field_ref : v->struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_rvect(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + stack_offset += stack_width; + + int tensor_offset = 0; + bool exists_in_literal = false; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + int tensor_item_width = v_field->field_ref->declared_type->get_width_on_stack(); + if (v_field->get_field_name() == field_ref->name) { + exists_in_literal = true; + std::vector literal_field_rvect(literal_rvect.begin() + tensor_offset, literal_rvect.begin() + tensor_offset + tensor_item_width); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(literal_field_rvect)); + break; + } + tensor_offset += tensor_item_width; + } + if (exists_in_literal || field_ref->declared_type == TypeDataNever::create()) { + continue; + } + + tolk_assert(field_ref->has_default_value()); + SrcLocation last_loc = v->get_body()->empty() ? v->loc : v->get_body()->get_all_fields().back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + std::vector def_rvect = pre_compile_expr(v_force_loc, code, field_ref->declared_type); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(def_rvect)); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_object_literal(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // an object (an instance of a struct) is actually a tensor at low-level + // for example, `struct User { id: int; name: slice; }` occupies 2 slots + // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + bool are_fields_shuffled = false; + for (int i = 1; i < v->get_body()->get_num_fields(); ++i) { + StructFieldPtr field_ref = v->struct_ref->find_field(v->get_body()->get_field(i)->get_field_name()); + StructFieldPtr prev_field_ref = v->struct_ref->find_field(v->get_body()->get_field(i - 1)->get_field_name()); + are_fields_shuffled |= prev_field_ref->field_idx > field_ref->field_idx; + } + + // if fields are created {y,x} (not {x,y}), maybe, it's nevertheless safe to evaluate them as {x,y}; + // for example, if they are just constants, calls to pure non-mutating functions, etc.; + // generally, rules of "can we evaluate {x,y} instead of {y,x}" follows the same logic + // as passing of calling `f(x,y)` with asm arg_order, is it safe to avoid SWAP + if (are_fields_shuffled && !CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v->get_body())) { + // okay, we have `{y: getY(), x: getX()}` / `{y: v += 1, x: v}`, evaluate them in created order + return process_object_literal_shuffled(v, code, target_type, lval_ctx); + } + + SrcLocation prev_loc = v->loc; + std::vector tensor_items; + std::vector target_types; + tensor_items.reserve(v->struct_ref->get_num_fields()); + target_types.reserve(v->struct_ref->get_num_fields()); + for (StructFieldPtr field_ref : v->struct_ref->fields) { + AnyExprV v_init_val = nullptr; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + if (v_field->get_field_name() == field_ref->name) { + v_init_val = v_field->get_init_val(); + break; + } + } + if (!v_init_val) { + if (field_ref->declared_type == TypeDataNever::create()) { + continue; // field of `never` type can be missed out of object literal (useful in generics defaults) + } // (it occupies 0 slots, nothing is assignable to it — like this field is missing from a struct) + tolk_assert(field_ref->has_default_value()); + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + v_init_val = v_force_loc; // it may be a complex expression, but it's a constant, checked earlier + } + tensor_items.push_back(v_init_val); + target_types.push_back(field_ref->declared_type); + prev_loc = v_init_val->loc; + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + // here, like everywhere, even for just `int`, there might be a potential transition due to union types + // example: passing `1` to `int | slice` puts actually "1 5" on a stack (1 for value, 5 for UTag = type_id of `int`) return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - ConstantValue value = eval_const_init_value(v); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - if (value.is_int()) { - code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); - } else { - code.emplace_back(v->loc, Op::_SliceConst, rvect, value.as_slice()); - } + code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol(v->bool_val ? "__true" : "__false")->try_as(); + FunctionPtr builtin_sym = lookup_function(v->bool_val ? "__true" : "__false"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr builtin_sym = lookup_function("__null"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1027,6 +1925,7 @@ static std::vector process_local_var(V v, CodeBlob static std::vector process_local_vars_declaration(V, CodeBlob&) { // it can not appear as a standalone expression // `var ... = rhs` is handled by ast_assign + // `var rhs: int lateinit` is ast_local_var_lhs tolk_assert(false); } @@ -1035,8 +1934,91 @@ static std::vector process_underscore(V v, CodeBlob& return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); } +static std::vector process_empty_expression(V v, CodeBlob& code, TypePtr target_type) { + std::vector empty_rvect; + return transition_to_target_type(std::move(empty_rvect), code, target_type, v); +} + +static std::vector process_artificial_aux_vertex(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + if (const auto* data = dynamic_cast(v->aux_data)) { + SrcLocation backup = code.forced_loc; + code.forced_loc = data->forced_loc; + std::vector rvect = pre_compile_expr(v->get_wrapped_expr(), code, target_type, lval_ctx); + code.forced_loc = backup; + return rvect; + } + + // aux "load x"; example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` + if (const auto* data = dynamic_cast(v->aux_data)) { + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + + std::vector ir_obj = data->var_ref->ir_idx; // loading will update stack slots of `p` + TypePtr t_orig = data->var_ref->declared_type; + + if (data->field_ref) { // extract a field from a whole lazy variable + tolk_assert(lazy_variable->is_struct()); + int stack_offset = calc_offset_on_stack(lazy_variable->loaded_state.original_struct, data->field_ref->field_idx); + int stack_width = data->field_ref->declared_type->get_width_on_stack(); + ir_obj = std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + t_orig = data->field_ref->declared_type; + } + + if (data->union_variant) { // extract a variant from a union (a union variable or a union field of a struct) + ir_obj = transition_to_target_type(std::move(ir_obj), code, t_orig, data->union_variant, v->loc); + } + + // `load_info` contains instructions to skip, load, save tail, etc.; + // it generates LETs to ir_obj, so stack slots of lazy_variable will contain loaded data + generate_lazy_struct_from_slice(code, v->loc, lazy_variable, data->load_info, ir_obj); + return transition_to_target_type({}, code, target_type, v); + } + + // aux "match(lazyUnion)" / aux "match(obj.lastUnionField)" + if (const auto* data = dynamic_cast(v->aux_data)) { + V v_match = v->get_wrapped_expr()->as(); + pre_compile_expr(v_match->get_subject(), code, nullptr); + + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + TypePtr t_union = data->field_ref ? data->field_ref->declared_type : data->var_ref->declared_type; + + std::vector match_blocks; + match_blocks.reserve(v_match->get_arms_count()); + for (int i = 0; i < v_match->get_arms_count(); ++i) { + auto v_arm = v_match->get_arm(i); + TypePtr arm_variant = nullptr; + if (v_arm->pattern_kind == MatchArmKind::exact_type) { + arm_variant = v_arm->pattern_type_node->resolved_type->unwrap_alias(); + } else { + tolk_assert(v_arm->pattern_kind == MatchArmKind::else_branch); // `else` allowed in a lazy match + } + match_blocks.emplace_back(LazyMatchOptions::MatchBlock{arm_variant, v_arm->get_body(), v_arm->get_body()->inferred_type}); + } + + LazyMatchOptions options = { + .match_expr_type = v->inferred_type, + .is_statement = v_match->is_statement(), + .add_return_to_all_arms = v == stmt_before_immediate_return, + .match_blocks = std::move(match_blocks), + }; + + // it will generate match by a slice prefix, and for each `match` arm, invoke pre_compile_expr(), + // which contains "aux load" particularly + std::vector ir_match = generate_lazy_match_for_union(code, v->loc, t_union, lazy_variable, options); + return transition_to_target_type(std::move(ir_match), code, target_type, v); + } + + if (const auto* data = dynamic_cast(v->aux_data)) { + std::vector rvect = data->generate_get_InMessage_field(code, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + tolk_assert(false); +} + std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - switch (v->type) { + switch (v->kind) { case ast_reference: return process_reference(v->as(), code, target_type, lval_ctx); case ast_assign: @@ -1051,20 +2033,28 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_ternary_operator(v->as(), code, target_type); case ast_cast_as_operator: return process_cast_as_operator(v->as(), code, target_type, lval_ctx); + case ast_is_type_operator: + return process_is_type_operator(v->as(), code, target_type); case ast_not_null_operator: return process_not_null_operator(v->as(), code, target_type, lval_ctx); - case ast_is_null_check: - return process_is_null_check(v->as(), code, target_type); + case ast_lazy_operator: + return process_lazy_operator(v->as(), code, target_type); + case ast_match_expression: + return process_match_expression(v->as(), code, target_type); case ast_dot_access: return process_dot_access(v->as(), code, target_type, lval_ctx); case ast_function_call: return process_function_call(v->as(), code, target_type); case ast_parenthesized_expression: return pre_compile_expr(v->as()->get_expr(), code, target_type, lval_ctx); + case ast_braced_expression: + return process_braced_expression(v->as(), code, target_type); case ast_tensor: return process_tensor(v->as(), code, target_type, lval_ctx); - case ast_typed_tuple: - return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_bracket_tuple: + return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_object_literal: + return process_object_literal(v->as(), code, target_type, lval_ctx); case ast_int_const: return process_int_const(v->as(), code, target_type); case ast_string_const: @@ -1079,34 +2069,51 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_local_vars_declaration(v->as(), code); case ast_underscore: return process_underscore(v->as(), code); + case ast_empty_expression: + return process_empty_expression(v->as(), code, target_type); + case ast_artificial_aux_vertex: + return process_artificial_aux_vertex(v->as(), code, target_type, lval_ctx); default: - throw UnexpectedASTNodeType(v, "pre_compile_expr"); + throw UnexpectedASTNodeKind(v, "pre_compile_expr"); } } -static void process_sequence(V v, CodeBlob& code) { - for (AnyV item : v->get_items()) { - process_any_statement(item, code); +static void process_block_statement(V v, CodeBlob& code) { + if (v->empty()) { + return; } + + FunctionPtr cur_f = code.fun_ref; + bool does_f_return_nothing = cur_f->inferred_return_type == TypeDataVoid::create() && !cur_f->does_return_self() && !cur_f->has_mutate_params(); + bool is_toplevel_block = v == cur_f->ast_root->as()->get_body(); + bool inlining_doesnt_prevent = code.inline_rvect_out == nullptr || code.inlining_before_immediate_return; + + // we want to optimize `match` and `if/else`: if it's the last statement, implicitly add "return" to every branch + // (to generate IFJMP instead of nested IF ELSE); + // a competent way is to do it at the IR level (building CST, etc.), it's impossible to tweak Ops for now; + // so, for every `f() { here }` of `... here; return;`, save it into a global, and handle within match/if + AnyV backup = stmt_before_immediate_return; + for (int i = 0; i < v->size() - 1; ++i) { + AnyV stmt = v->get_item(i); + AnyV next_stmt = v->get_item(i + 1); + bool next_is_empty_return = next_stmt->kind == ast_return_statement && !next_stmt->as()->has_return_value(); + stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing && inlining_doesnt_prevent ? stmt : nullptr; + process_any_statement(stmt, code); + } + AnyV last_stmt = v->get_item(v->size() - 1); + stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing && inlining_doesnt_prevent ? last_stmt : nullptr; + process_any_statement(last_stmt, code); + stmt_before_immediate_return = backup; } static void process_assert_statement(V v, CodeBlob& code) { - std::vector args(3); - if (auto v_not = v->get_cond()->try_as(); v_not && v_not->tok == tok_logical_not) { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond()->as()->get_rhs(); - args[2] = createV(v->loc, true); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } else { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond(); - args[2] = createV(v->loc, false); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } + std::vector ir_thrown_code = pre_compile_expr(v->get_thrown_code(), code); + std::vector ir_cond = pre_compile_expr(v->get_cond(), code); + tolk_assert(ir_cond.size() == 1 && ir_thrown_code.size() == 1); - FunctionPtr builtin_sym = lookup_global_symbol("__throw_if_unless")->try_as(); - std::vector args_vars = pre_compile_tensor(code, args); + std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v->loc, 0, "(assert-0)") }; + FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } @@ -1160,9 +2167,15 @@ static void process_if_statement(V v, CodeBlob& code) { Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); code.push_set_cur(if_op.block0); process_any_statement(v->get_if_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_if_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_if_body()->loc_end); code.push_set_cur(if_op.block1); process_any_statement(v->get_else_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_else_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_else_body()->loc_end); if (v->is_ifnot) { std::swap(if_op.block0, if_op.block1); @@ -1194,16 +2207,16 @@ static void process_do_while_statement(V v, CodeBlob& co until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); - } else if (cond->inferred_type == TypeDataBool::create()) { + } else if (cond->inferred_type->unwrap_alias() == TypeDataBool::create()) { until_cond = createV(cond->loc, "!b", tok_logical_not, cond); } else { until_cond = createV(cond->loc, "!", tok_logical_not, cond); } until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { - v_bin->mutate()->assign_fun_ref(lookup_global_symbol("_" + static_cast(v_bin->operator_name) + "_")->try_as()); + v_bin->mutate()->assign_fun_ref(lookup_function("_" + static_cast(v_bin->operator_name) + "_")); } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { - v_un->mutate()->assign_fun_ref(lookup_global_symbol(static_cast(v_un->operator_name) + "_")->try_as()); + v_un->mutate()->assign_fun_ref(lookup_function(static_cast(v_un->operator_name) + "_")); } until_op.left = pre_compile_expr(until_cond, code, nullptr); @@ -1224,38 +2237,46 @@ static void process_while_statement(V v, CodeBlob& code) { static void process_throw_statement(V v, CodeBlob& code) { if (v->has_thrown_arg()) { - FunctionPtr builtin_sym = lookup_global_symbol("__throw_arg")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw_arg"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__throw")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } } static void process_return_statement(V v, CodeBlob& code) { - std::vector return_vars; - if (v->has_return_value()) { - TypePtr child_target_type = code.fun_ref->inferred_return_type; - if (code.fun_ref->does_return_self()) { - child_target_type = code.fun_ref->parameters[0].declared_type; - } - return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + // it's a function we're traversing AST of; + // probably, it's called and inlined into another (outer) function, we handle this below + FunctionPtr fun_ref = code.fun_ref; + + TypePtr child_target_type = fun_ref->inferred_return_type; + if (fun_ref->does_return_self()) { + child_target_type = fun_ref->parameters[0].declared_type; } - if (code.fun_ref->does_return_self()) { + std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + + if (fun_ref->does_return_self()) { return_vars = {}; } - if (code.fun_ref->has_mutate_params()) { + if (fun_ref->has_mutate_params()) { std::vector mutated_vars; - for (const LocalVarData& p_sym: code.fun_ref->parameters) { + for (const LocalVarData& p_sym: fun_ref->parameters) { if (p_sym.is_mutate_parameter()) { mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } return_vars.insert(return_vars.begin(), mutated_vars.begin(), mutated_vars.end()); } - code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + + // if fun_ref is called and inlined into a parent, assign a result instead of generating a return statement + if (code.inline_rvect_out) { + code.emplace_back(v->loc, Op::_Let, *code.inline_rvect_out, std::move(return_vars)); + } else { + code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + } } // append "return" (void) to the end of the function @@ -1276,9 +2297,9 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code void process_any_statement(AnyV v, CodeBlob& code) { - switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), code); + switch (v->kind) { + case ast_block_statement: + return process_block_statement(v->as(), code); case ast_return_statement: return process_return_statement(v->as(), code); case ast_repeat_statement: @@ -1303,8 +2324,8 @@ void process_any_statement(AnyV v, CodeBlob& code) { } static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { - auto v_body = fun_ref->ast_root->as()->get_body()->as(); - CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; + auto v_body = fun_ref->ast_root->as()->get_body()->as(); + CodeBlob* blob = new CodeBlob(fun_ref); std::vector rvect_import; int total_arg_width = 0; @@ -1323,9 +2344,11 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); - for (AnyV item : v_body->get_items()) { - process_any_statement(item, *blob); + if (fun_ref->name == "onInternalMessage") { + handle_onInternalMessage_codegen_start(fun_ref, rvect_import, *blob, fun_ref->loc); } + + process_block_statement(v_body, *blob); append_implicit_return_statement(v_body->loc_end, *blob); blob->close_blk(v_body->loc_end); @@ -1343,7 +2366,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ for (char c : ops) { if (c == '\n' || c == '\r') { if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -1354,7 +2377,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ } } if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -1421,7 +2444,7 @@ class ConvertASTToLegacyOpVisitor final { static void start_visiting_function(FunctionPtr fun_ref, V) { tolk_assert(fun_ref->is_type_inferring_done()); - if (fun_ref->is_code_function()) { + if (fun_ref->is_code_function() && !fun_ref->is_inlined_in_place()) { convert_function_body_to_CodeBlob(fun_ref, std::get(fun_ref->body)); } else if (fun_ref->is_asm_function()) { convert_asm_body_to_AsmOp(fun_ref, std::get(fun_ref->body)); diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index 1f374bc81..dc269a342 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -24,6 +24,8 @@ * * Example: `a = b`, `a` is lvalue, `b` is rvalue. * Example: `a + b`, both are rvalue. + * Example: `a;`, it's none (not rvalue, not lvalue). + * For instance, difference between none and rvalue helps detect whether `match` is expression or statement. * * Note, that this pass only assigns, not checks. So, for `f() = 4`, expr `f()` is lvalue. * Checking (firing this as incorrect later) is performed after type inferring, see pipe-check-rvalue-lvalue. @@ -47,79 +49,108 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { return saved; } + MarkingState enter_rvalue_if_none() { + if (cur_state != MarkingState::None) { + return cur_state; + } + cur_state = MarkingState::RValue; + return MarkingState::None; + } + void restore_state(MarkingState saved) { cur_state = saved; } - void mark_vertex_cur_or_rvalue(AnyExprV v) const { + void mark_vertex(AnyExprV v) const { if (cur_state == MarkingState::LValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_lvalue_true(); } - if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue || cur_state == MarkingState::None) { + if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_rvalue_true(); } } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::None); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } - void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(v->passed_as_mutate ? MarkingState::LValueAndRValue : MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); parent::visit(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v->get_obj()); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); @@ -130,11 +161,11 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { // so, if current state is "lvalue", `_` will be marked as lvalue, and ok // but if used incorrectly, like `f(_)` or just `_;`, it will be marked rvalue // and will fire an error later, in pipe lvalue/rvalue check - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -143,7 +174,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValueAndRValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -152,53 +183,140 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); // both cond, when_true and when_false are rvalue, `(cond ? a : b) = 5` prohibited restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate (t.0 as int)` both `t.0 as int` and `t.0` are lvalue } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_expr()); + restore_state(saved); + } + void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue } - void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + void visit(V v) override { + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); - parent::visit(v->get_expr()); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); restore_state(saved); } void visit(V v) override { - tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + tolk_assert(cur_state == MarkingState::LValue || v->is_lateinit); + mark_vertex(v); parent::visit(v); } void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::None); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); + restore_state(saved); } void visit(V v) override { diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index bae67c5fd..97a953f4f 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -31,6 +31,11 @@ static std::string to_string(AnyExprV v_with_type) { return "`" + v_with_type->inferred_type->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); +} + GNU_ATTRIBUTE_NOINLINE static std::string expression_as_string(AnyExprV v) { if (auto v_ref = v->try_as()) { @@ -44,10 +49,36 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } -// fire a general "type mismatch" error, just a wrapper over `throw` +// fire a general error on type mismatch; for example, "can not assign `cell` to `slice`"; +// for instance, if `as` operator is applicable, compiler will suggest it GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); +static void fire_error_type_mismatch(FunctionPtr cur_f, SrcLocation loc, const char* text_tpl, TypePtr src, TypePtr dst) { +#ifdef TOLK_DEBUG + tolk_assert(!dst->can_rhs_be_assigned(src)); +#endif + std::string message = text_tpl; + message.replace(message.find("{src}"), 5, to_string(src)); + message.replace(message.find("{dst}"), 5, to_string(dst)); + if (src->can_be_casted_with_as_operator(dst)) { + bool suggest_as = !dst->try_as() && !dst->try_as(); + if ((src == TypeDataAddress::create() || src == TypeDataSlice::create()) && (dst == TypeDataAddress::create() || dst == TypeDataSlice::create())) { + message += "\nhint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; + message += "\n most likely, you just need `address` everywhere"; + message += "\nhint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } else if (suggest_as) { + message += "\nhint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } + if (src == TypeDataBool::create() && dst == TypeDataInt::create()) { + message += "\ncaution! in TVM, bool TRUE is -1, not 1"; + } + } + if (const TypeDataUnion* src_nullable = src->try_as(); src_nullable && src_nullable->or_null) { + if (dst->can_rhs_be_assigned(src_nullable->or_null)) { + message += "\nhint: probably, you should check on null"; + message += "\nhint: alternatively, use `!` operator to bypass nullability checks: `!`"; + } + } + fire(cur_f, loc, message); } // fire an error on `!cell` / `+slice` @@ -66,6 +97,10 @@ static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, GNU_ATTRIBUTE_NOINLINE static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { + bool no_warning = cond->kind == ast_bool_const || cond->kind == ast_int_const; + if (no_warning) { // allow `while(true)` without a warning + return; + } loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); } @@ -75,7 +110,7 @@ static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type if (is_obj_of_dot_call) { fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); } else { - fire(cur_f, ith_arg->loc, "can not pass " + to_string(ith_arg) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, ith_arg->loc, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); } } } @@ -99,7 +134,7 @@ static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { std::string var_name = assigned_var->name; - fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null; specify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); + fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); } // fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple @@ -117,22 +152,58 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vname == "__expect_type") { tolk_assert(v->get_num_args() == 2); - TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); + std::string_view expected_type_str = v->get_arg(1)->get_expr()->as()->str_val; TypePtr expr_type = v->get_arg(0)->inferred_type; - if (expected_type != expr_type) { - fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); + if (expected_type_str != expr_type->as_human_readable()) { + fire(cur_f, v->loc, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); } } } +static bool expect_integer(TypePtr inferred_type) { + if (inferred_type == TypeDataInt::create()) { + return true; + } + if (inferred_type->try_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; +} + static bool expect_integer(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataInt::create(); + return expect_integer(v_inferred->inferred_type); +} + +static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; } static bool expect_boolean(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataBool::create(); + return expect_boolean(v_inferred->inferred_type); } +static bool expect_address(TypePtr inferred_type) { + if (inferred_type == TypeDataAddress::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_address(as_alias->underlying_type); + } + return false; +} + +static bool expect_address(AnyExprV v_inferred) { + return expect_address(v_inferred->inferred_type); +} class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value @@ -145,6 +216,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(rhs); // all operators (+=, etc.) can work for integers (if both sides are integers) + // for intN, they are also allowed (int16 |= int8 is ok, since int16 | int8 is ok, all arithmetic is int) bool types_ok = expect_integer(lhs) && expect_integer(rhs); // bitwise operators &= |= ^= are "overloaded" for booleans also (if both sides are booleans) if (!types_ok && (v->tok == tok_set_bitwise_and || v->tok == tok_set_bitwise_or || v->tok == tok_set_bitwise_xor)) { @@ -181,14 +253,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v->tok) { // == != can compare both integers and booleans, (int == bool) is NOT allowed + // for intN, it also works: (int8 == int16) is ok, (int == uint32) is ok // note, that `int?` and `int?` can't be compared, since Fift `EQUAL` works with integers only - // (if to allow `int?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) + // (if to allow `int?`/`int8?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) case tok_eq: case tok_neq: { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - if (lhs->inferred_type == rhs->inferred_type) { // compare slice with slice, int? with int? + bool both_address = expect_address(lhs) && expect_address(rhs); + if (both_address) { // address can be compared with ==, but it's not integer comparison, it's handled specially + v->mutate()->assign_fun_ref(nullptr); + } else if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); @@ -207,6 +283,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } break; // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed + // they are allowed for intN (int16 & int32 is ok) and always "fall back" to general int case tok_bitwise_and: case tok_bitwise_or: case tok_bitwise_xor: { @@ -228,6 +305,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { break; } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: if (!expect_integer(lhs) || !expect_integer(rhs)) { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); @@ -238,33 +316,38 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v->get_expr()); - if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->cast_to_type)) { - fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->cast_to_type)); + if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->type_node->resolved_type)) { + fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); } } - void visit(V v) override { + void visit(V v) override { parent::visit(v->get_expr()); + TypePtr rhs_type = v->type_node->resolved_type; - if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { - // operator `!` used for always-null (proven by smart casts, for example), it's an error - fire(cur_f, v->loc, "operator `!` used for always null expression"); + if (rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed + fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); } - // if operator `!` used for non-nullable, probably a warning should be printed - } - - void visit(V v) override { - parent::visit(v->get_expr()); if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " is always not null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { + // operator `!` used for always-null (proven by smart casts, for example), it's an error + fire(cur_f, v->loc, "operator `!` used for always null expression"); } + // if operator `!` used for non-nullable, probably a warning should be printed } - void visit(V v) override { + void visit(V v) override { parent::visit(v); for (int i = 0; i < v->size(); ++i) { @@ -278,9 +361,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v); - TypePtr obj_type = v->get_obj()->inferred_type; if (v->is_target_indexed_access()) { - if (obj_type->try_as() && v->inferred_type->get_width_on_stack() != 1) { + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); + if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -292,32 +375,28 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr fun_ref = v->fun_maybe; if (!fun_ref) { // `local_var(args)` and similar - const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->try_as(); + const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->unwrap_alias()->try_as(); tolk_assert(f_callable && f_callable->params_size() == v->get_num_args()); for (int i = 0; i < v->get_num_args(); ++i) { auto arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = f_callable->params_types[i]; if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { - fire(cur_f, arg_i->loc, "can not pass " + to_string(arg_i) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, arg_i->loc, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); } } return; } - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - int delta_self = 0; - AnyExprV dot_obj = nullptr; - if (auto v_dot = v->get_callee()->try_as()) { - delta_self = 1; - dot_obj = v_dot->get_obj(); - } + // so, we have a call `f(args)` or `obj.f(args)`, fun_ref is a function/method (code / asm / builtin) + AnyExprV self_obj = v->get_self_obj(); + int delta_self = self_obj != nullptr; - if (dot_obj) { + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; - check_function_argument_passed(cur_f, param_type, dot_obj, true); + check_function_argument_passed(cur_f, param_type, self_obj, true); if (param_0.is_mutate_parameter()) { - check_function_argument_mutate_back(cur_f, param_type, dot_obj, true); + check_function_argument_mutate_back(cur_f, param_type, self_obj, true); } } for (int i = 0; i < v->get_num_args(); ++i) { @@ -357,18 +436,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->declared_type; // `var v: int = rhs` (otherwise, nullptr) + TypePtr declared_type = lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->marked_as_redef) { tolk_assert(lhs_var->var_ref && lhs_var->var_ref->declared_type); declared_type = lhs_var->var_ref->declared_type; } if (declared_type) { if (!declared_type->can_rhs_be_assigned(rhs_type)) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(declared_type)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); } } else { if (rhs_type == TypeDataNullLiteral::create()) { - fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->type == ast_null_keyword); + fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); } } return; @@ -377,7 +456,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tensor) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); } @@ -393,15 +472,15 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } if (lhs_tuple->size() != rhs_type_tuple->size()) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); } - V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tuple->size(); ++i) { process_assignment_lhs(lhs_tuple->get_item(i), rhs_type_tuple->items[i], rhs_tuple_maybe ? rhs_tuple_maybe->get_item(i) : nullptr); } @@ -410,7 +489,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // check `untypedTuple.0 = rhs_tensor` and other non-1 width elements if (auto lhs_dot = lhs->try_as()) { - if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type == TypeDataTuple::create()) { + if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type->unwrap_alias() == TypeDataTuple::create()) { if (rhs_type->get_width_on_stack() != 1) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); } @@ -422,9 +501,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); + } else if (lhs->try_as()) { + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); } else { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); } } } @@ -441,18 +522,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr expr_type = v->get_return_value()->inferred_type; if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { - fire(cur_f, v->get_return_value()->loc, "can not convert type " + to_string(expr_type) + " to return type " + to_string(cur_f->inferred_return_type)); + fire_error_type_mismatch(cur_f, v->get_return_value()->loc, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); } } static bool is_expr_valid_as_return_self(AnyExprV return_expr) { // `return self` - if (return_expr->type == ast_reference && return_expr->as()->get_name() == "self") { + if (return_expr->kind == ast_reference && return_expr->as()->get_name() == "self") { return true; } // `return self.someMethod()` - if (auto v_call = return_expr->try_as(); v_call && v_call->is_dot_call()) { - return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_dot_obj()); + if (auto v_call = return_expr->try_as(); v_call && v_call->get_self_obj()) { + return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_self_obj()); } // `return cond ? ... : ...` if (auto v_ternary = return_expr->try_as()) { @@ -474,6 +555,113 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + parent::visit(v); + + bool has_type_arm = false; + bool has_expr_arm = false; + bool has_else_arm = false; + AnyExprV v_subject = v->get_subject(); + TypePtr subject_type = v_subject->inferred_type->unwrap_alias(); + const TypeDataUnion* subject_union = subject_type->try_as(); + + std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + switch (v_arm->pattern_kind) { + case MatchArmKind::exact_type: { + if (has_expr_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_type_arm = true; + + TypePtr lhs_type = v_arm->pattern_type_node->resolved_type->unwrap_alias(); // `lhs_type => ...` + if (lhs_type->try_as()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); + } + bool can_happen = (subject_union && subject_union->has_variant_with_type_id(lhs_type)) || + (!subject_union && subject_type->equal_to(lhs_type)); + if (!can_happen) { + fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); + } + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), lhs_type->get_type_id()) != covered_type_ids.end()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); + } + covered_type_ids.push_back(lhs_type->get_type_id()); + break; + } + case MatchArmKind::const_expression: { + if (has_type_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_expr_arm = true; + TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type->unwrap_alias(); + bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); + bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); + if (!both_int && !both_bool) { + if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); + } else { + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); + } + } + break; + } + default: + if (has_else_arm) { + fire(cur_f, v_arm->loc, "duplicated `else` branch"); + } + if (has_type_arm) { + // `else` is not allowed in `match` by type, but we don't fire an error here, + // because it might turn out to be a lazy `match`, where `else` is allowed; + // if it's not lazy, an error is fired later + } + has_else_arm = true; + } + } + + // fire if `match` by type is not exhaustive + if (has_type_arm && subject_union && subject_union->variants.size() != covered_type_ids.size()) { + std::string missing; + for (TypePtr variant : subject_union->variants) { + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), variant->get_type_id()) == covered_type_ids.end()) { + if (!missing.empty()) { + missing += ", "; + } + missing += to_string(variant); + } + } + fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + } + // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) + if (has_expr_arm && !has_else_arm && !v->is_statement()) { + bool needs_else_branch = true; + if (expect_boolean(subject_type) && v->get_arms_count() == 2) { + auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); + auto arm1 = v->get_arm(1)->get_pattern_expr()->try_as(); + needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); + } + if (needs_else_branch) { + fire(cur_f, v->loc, "`match` expression should have `else` branch"); + } + } + } + + void visit(V v) override { + parent::visit(v->get_init_val()); + + if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { + fire_error_type_mismatch(cur_f, v->get_init_val()->loc, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); + } + } + void visit(V v) override { parent::visit(v); @@ -549,7 +737,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { + void visit(V v) override { parent::visit(v); if (v->first_unreachable) { @@ -573,14 +761,63 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + } + + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + + TypePtr inferred_type = param_ref->default_value->inferred_type; + if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + fire_error_type_mismatch(fun_ref, param_ref->loc, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); + } + } + } + } + + // given `const a = 2 + 3` check types within its init_value + // so, `const a = 1 + some_slice` will fire a reasonable error + void start_visiting_constant(GlobalConstPtr const_ref) { + parent::visit(const_ref->init_value); + + // if no errors occurred, init value has correct type + // (though it may not be a valid constant expression, this would be checked after type inferring) + if (const_ref->declared_type) { // `const a: int = ...` + TypePtr inferred_type = const_ref->init_value->inferred_type; + if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + fire_error_type_mismatch(nullptr, const_ref->loc, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); } } } + + // given struct field `a: int = 2 + 3` check types within its default_value + void start_visiting_field_default(StructFieldPtr field_ref) { + parent::visit(field_ref->default_value); + + TypePtr inferred_type = field_ref->default_value->inferred_type; + if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); + } + } }; void pipeline_check_inferred_types() { visit_ast_of_all_functions(); + + CheckInferredTypesVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + visitor.start_visiting_constant(const_ref); + } + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { + visitor.start_visiting_field_default(field_ref); + } + } + } } } // namespace tolk diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 366ff1607..1a9143875 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -27,15 +27,17 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_impure_operation_inside_pure_function(AnyV v) { - v->error("an impure operation in a pure function"); +static void fire_error_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcLocation loc) { + fire(cur_f, loc, "an impure operation in a pure function"); } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { - static void fire_if_global_var(AnyExprV v) { + FunctionPtr cur_f = nullptr; + + void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { if (v_ident->sym->try_as()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } } } @@ -54,11 +56,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` if (!v->fun_maybe) { // `local_var(args)` is always impure, no considerations about what's there at runtime - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } if (!v->fun_maybe->is_marked_as_pure()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } parent::visit(v); @@ -73,17 +75,22 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_check_pure_impure_operations() { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 3ec47a16b..cb8067628 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -30,64 +30,80 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_be_used_as_lvalue(AnyV v, const std::string& details) { +static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { // example: `f() = 32` // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) - v->error(details + " can not be used as lvalue"); + throw ParseError(cur_f, v->loc, details + " can not be used as lvalue"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(AnyExprV v, LocalVarPtr var_ref) { +static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - v->error("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + throw ParseError(cur_f, loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - v->error("modifying immutable variable `" + var_ref->name + "`"); + throw ParseError(cur_f, loc, "modifying immutable variable `" + var_ref->name + "`"); } } // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions -static void validate_function_used_as_noncall(AnyExprV v, FunctionPtr fun_ref) { +static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { - v->error("saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); } if (fun_ref->has_mutate_params()) { - v->error("saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); } } class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + void on_var_used_as_lvalue(SrcLocation loc, LocalVarPtr var_ref) const { + if (var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(cur_f, loc, var_ref); + } + var_ref->mutate()->assign_used_as_lval(); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator ?:"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); } parent::visit(v); } @@ -97,60 +113,92 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); + } + parent::visit(v->get_expr()); + } + void visit(V v) override { // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok parent::visit(v->get_expr()); } - void visit(V v) override { + void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, v->is_negated ? "operator !=" : "operator =="); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); } parent::visit(v->get_expr()); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { + // check for `immutableVal.field = rhs` or any other mutation of an immutable tensor/tuple/object + // don't allow cheating like `((immutableVal!)).field = rhs` + if (v->is_lvalue) { + AnyExprV leftmost_obj = v->get_obj(); + while (true) { + if (auto as_dot = leftmost_obj->try_as()) { + leftmost_obj = as_dot->get_obj(); + } else if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; + } + } + + if (auto as_ref = leftmost_obj->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + on_var_used_as_lvalue(leftmost_obj->loc, var_ref); + } + } + } + // a reference to a method used as rvalue, like `var v = t.tupleAt` if (v->is_rvalue && v->is_target_fun_ref()) { - validate_function_used_as_noncall(v, std::get(v->target)); + validate_function_used_as_noncall(cur_f, v, std::get(v->target)); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "function call"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "function call"); } if (!v->fun_maybe) { parent::visit(v->get_callee()); } // for `f()` don't visit ast_reference `f`, to detect `f` usage as non-call, like `var cb = f` // same for `obj.method()`, don't visit ast_reference method, visit only obj - if (v->is_dot_call()) { - parent::visit(v->get_dot_obj()); + if (AnyExprV self_obj = v->get_self_obj()) { + parent::visit(self_obj); } for (int i = 0; i < v->get_num_args(); ++i) { @@ -158,11 +206,18 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->marked_as_redef) { tolk_assert(v->var_ref); if (v->var_ref->is_immutable()) { - v->error("`redef` for immutable variable"); + fire(cur_f, v->loc, "`redef` for immutable variable"); } } } @@ -170,24 +225,24 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { tolk_assert(v->sym); - if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(v, var_ref); + if (LocalVarPtr var_ref = v->sym->try_as()) { + on_var_used_as_lvalue(v->loc, var_ref); } else if (v->sym->try_as()) { - v->error("modifying immutable constant"); + fire(cur_f, v->loc, "modifying immutable constant"); } else if (v->sym->try_as()) { - v->error("function can't be used as lvalue"); + fire(cur_f, v->loc, "function can't be used as lvalue"); } } // a reference to a function used as rvalue, like `var v = someFunction` if (FunctionPtr fun_ref = v->sym->try_as(); fun_ref && v->is_rvalue) { - validate_function_used_as_noncall(v, fun_ref); + validate_function_used_as_noncall(cur_f, v, fun_ref); } } void visit(V v) override { if (v->is_rvalue) { - v->error("`_` can't be used as a value; it's a placeholder for a left side of assignment"); + fire(cur_f, v->loc, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); } } @@ -201,6 +256,12 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + } }; void pipeline_check_rvalue_lvalue() { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp new file mode 100644 index 000000000..f8b9b9cd0 --- /dev/null +++ b/tolk/pipe-check-serialized-fields.cpp @@ -0,0 +1,117 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "type-system.h" + +namespace tolk { + +// fire an error on overflow 1023 bits +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { + throw ParseError(struct_ref->ast_root->loc, + "struct `" + struct_ref->as_human_readable() + "` can exceed 1023 bits in serialization (estimated size: " + std::to_string(size.min_bits) + ".." + std::to_string(size.max_bits) + " bits)\n\n" + "1) either suppress it by adding an annotation:\n" + "> @overflow1023_policy(\"suppress\")\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> }\n" + " then, if limit exceeds, it will fail at runtime: you've manually agreed to ignore this\n\n" + "2) or place some fields into a separate struct (e.g. ExtraFields), and create a ref:\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> more: Cell;\n" + "> }\n" + ); +} + + +class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static void check_type_fits_cell_or_has_policy(TypePtr serialized_type) { + if (const TypeDataStruct* s_struct = serialized_type->unwrap_alias()->try_as()) { + check_struct_fits_cell_or_has_policy(s_struct); + } else if (const TypeDataUnion* s_union = serialized_type->unwrap_alias()->try_as()) { + for (TypePtr variant : s_union->variants) { + check_type_fits_cell_or_has_policy(variant); + } + } + } + + static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { + StructPtr struct_ref = t_struct->struct_ref; + bool avoid_check = struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "UnsafeBodyNoRef"; + if (avoid_check) { + return; + } + + PackSize size = estimate_serialization_size(t_struct); + if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { + if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { + fire_error_theoretical_overflow_1023(struct_ref, size); + } + } + // don't check Cell fields for overflow of T: it would be checked on load() or other interaction with T + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref || !fun_ref->is_compile_time_special_gen() || !fun_ref->is_instantiation_of_generic_function()) { + return; + } + + std::string_view f_name = fun_ref->base_fun_ref->name; + TypePtr serialized_type = nullptr; + bool is_pack = false; + if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.loadAny" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || + f_name == "createMessage" || f_name == "createExternalLogMessage") { + serialized_type = fun_ref->substitutedTs->typeT_at(0); + is_pack = f_name == "T.toCell" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; + } else { + return; // not a serialization function + } + + std::string because_msg; + if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { + std::string via_name = fun_ref->is_method() ? fun_ref->method_name : fun_ref->base_fun_ref->name; + fire(cur_f, v->loc, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + } + + check_type_fits_cell_or_has_policy(serialized_type); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_check_serialized_fields() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 9c27029b5..74fbb3453 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-replacer.h" #include "type-system.h" +#include "constant-evaluator.h" /* * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. @@ -57,12 +58,20 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } + static V create_string_const(SrcLocation loc, std::string&& literal_value, TypePtr inferred_type) { + auto v_string = createV(loc, literal_value); + v_string->assign_inferred_type(inferred_type); + v_string->assign_literal_value(std::move(literal_value)); + v_string->assign_rvalue_true(); + return v_string; + } + AnyExprV replace(V v) override { parent::replace(v); TokenType t = v->tok; // convert "-1" (tok_minus tok_int_const) to a const -1 - if (t == tok_minus && v->get_rhs()->type == ast_int_const) { + if (t == tok_minus && v->get_rhs()->kind == ast_int_const) { td::RefInt256 intval = v->get_rhs()->as()->intval; tolk_assert(!intval.is_null()); intval = -intval; @@ -72,40 +81,116 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return create_int_const(v->loc, std::move(intval)); } // same for "+1" - if (t == tok_plus && v->get_rhs()->type == ast_int_const) { + if (t == tok_plus && v->get_rhs()->kind == ast_int_const) { return v->get_rhs(); } // `!true` / `!false` - if (t == tok_logical_not && v->get_rhs()->type == ast_bool_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_bool_const) { return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); } // `!0` - if (t == tok_logical_not && v->get_rhs()->type == ast_int_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_int_const) { return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); } return v; } - AnyExprV replace(V v) override { + AnyExprV replace(V v) override { parent::replace(v); // `null == null` / `null != null` - if (v->get_expr()->type == ast_null_keyword) { + if (v->get_expr()->kind == ast_null_keyword && v->type_node->resolved_type == TypeDataNullLiteral::create()) { return create_bool_const(v->loc, !v->is_negated); } return v; } + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { + CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); + if (std::holds_alternative(value)) { + return create_int_const(v->loc, std::move(std::get(value))); + } else { + return create_string_const(v->loc, std::move(std::get(value)), v->fun_maybe->declared_return_type); + } + } + + return v; + } + + AnyExprV replace(V v) override { + // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, + // it's actually a slice + std::string literal_value = eval_string_const_standalone(v); + v->mutate()->assign_literal_value(std::move(literal_value)); + + return v; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `2 + 3 => ...` with `5 => ...` + // non-constant expressions like `foo() => ...` fire an error here + if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->kind != ast_int_const) { + check_expression_is_constant(v->get_pattern_expr()); + } + + return v; + } + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + check_expression_is_constant(param_ref->default_value); + AnyExprV replaced = replace_in_expression(param_ref->default_value); + param_ref->mutate()->assign_default_value(replaced); + } + } + + parent::replace(v_function->get_body()); + } + + // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. + AnyExprV replace_in_expression(AnyExprV init_value) { + return parent::replace(init_value); + } }; void pipeline_constant_folding() { + ConstantFoldingReplacer replacer; + + // here (after type inferring) check that `const a = 2 + 3` is a valid constant expression + // non-constant expressions like `const a = foo()` fire an error here + // also, replace `const a = ton("0.05")` with `const a = 50000000` + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + check_expression_is_constant(const_ref->init_value); + AnyExprV replaced = replacer.replace_in_expression(const_ref->init_value); + const_ref->mutate()->assign_init_value(replaced); + } + // do the same for default values of struct fields, they must be constant expressions + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { + check_expression_is_constant(field_ref->default_value); + AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); + field_ref->mutate()->assign_default_value(replaced); + } + } + } + replace_ast_of_all_functions(); } diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp new file mode 100644 index 000000000..7e892b687 --- /dev/null +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -0,0 +1,313 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include + +/* + * This pipe detects whether each function can be inlined in-place or not. + * Outcome: call `fun_ref->assign_inline_mode_in_place()` for "lightweight" or "called only once" functions, + * and they will be inlined in-place while converting AST to IR (to Ops), and won't be generated to Fift. + * + * Given AST only, there is no definite algorithm to predict whether a function is "simple" ("lightweight"), + * so that inlining it will do better. There are no correct metrics at AST level that can be mapped onto TVM complexity. + * So, instead of overcomplicating and fine-tuning an algorithm, we're heading a simple way: + * - if a function is tiny, inline it always + * - if a function is called only once, inline it (only there, obviously) + * - if a function is marked `@inline` (intended by the user), inline it in place (if possible) + * - see `should_auto_inline_if_not_prevented()` + * + * What can prevent a function from inlining? Several reasons: + * - it's recursive + * - it's used as non-call (a reference to it is taken) + * - see `is_inlining_prevented_even_if_annotated()` + * + * About `@inline` annotation. It means "user intention", so the compiler tries to inline it in-place + * without considering AST metrics. But anyway, something may prevent inlining (middle returns, for example). + * In this case, the desired flag is just not set; inline_mode remains inlineViaFif, we'll generate `PROCINLINE`. + * + * Besides inline detection, this pipe populates `fun_ref->n_times_called` (while building call graph). + * It's used in Fift output inside comments. + */ + +namespace tolk { + +// to calculate recursions, at first we populate the call graph +// (purpose: functions in recursive call chains can't be inlined) +static std::unordered_map> call_graph; + +static bool is_called_implicitly_by_compiler(FunctionPtr f) { + if (f->name == "onBouncedMessage") { + return true; + } + if (f->is_method() && f->method_name == "packToBuilder") { + return f->does_accept_self() && !f->does_mutate_self() && f->get_num_params() == 2 && f->has_mutate_params(); + } + if (f->is_method() && f->method_name == "unpackFromSlice") { + return !f->does_accept_self() && f->get_num_params() == 1 && f->has_mutate_params(); + } + return false; +} + +// when traversing a function, collect some AST metrics used to detect whether it's lightweight +struct StateWhileTraversingFunction { + FunctionPtr fun_ref; + bool has_returns_in_the_middle = false; + int n_statements = 0; + int n_function_calls = 0; + int n_binary_operators = 0; + int n_control_flow = 0; + int n_globals = 0; + int max_block_depth = 0; + + explicit StateWhileTraversingFunction(FunctionPtr fun_ref) + : fun_ref(fun_ref) {} + + int calculate_ast_cost() const { + return n_function_calls + n_binary_operators + n_statements * 2 + + n_control_flow * 10 + n_globals * 5 + (max_block_depth - 1) * 10; + } + + bool is_inlining_prevented_even_if_annotated() const { + // even if user specified `@inline`, we can't do anything about recursions, for example; + // in this case, in-place inlining won't happen, we'll generate `PROCINLINE` to Fift + bool is_inside_recursion = fun_ref->n_times_called >= 9999; + return has_returns_in_the_middle || is_inside_recursion || fun_ref->is_used_as_noncall() || !fun_ref->is_code_function(); + } + + bool should_auto_inline_if_not_prevented() const { + // if a function is called only once, inline it regardless of its size + // (to prevent this, `@inline_ref` can be used, for example) + if (fun_ref->n_times_called == 1) { + return true; + } + + // if a function is lightweight, inline in regardless of how many times it's called + // (for instance, `Storage.load` is always inlined) + int approx_cost_per_call = calculate_ast_cost(); + if (approx_cost_per_call < 30) { + return true; + } + + // try to _somehow_ detect whether to inline it or not + return approx_cost_per_call * fun_ref->n_times_called < 150; + } +}; + +// traverse the AST, collect metrics, and in the end, probably set the inline flag +class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { + StateWhileTraversingFunction cur_state{nullptr}; + int block_depth = 0; + std::vector> collected_expect_inline; // `__expect_inline()` compiler assertions + +protected: + void visit(V v) override { + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "__expect_inline") { + collected_expect_inline.push_back(v); + } else { + cur_state.n_function_calls++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->tok == tok_logical_and || v->tok == tok_logical_not) { + cur_state.n_control_flow++; + } else { + cur_state.n_binary_operators++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->sym->try_as()) { + cur_state.n_globals++; + } + } + + void visit(V v) override { + block_depth++; + cur_state.n_statements += v->size(); + cur_state.max_block_depth = std::max(cur_state.max_block_depth, block_depth); + parent::visit(v); + block_depth--; + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + // detect if `return` the last return statement in a function's body + // (currently in-place inlining for functions with returns in the middle is not supported) + auto body_block = cur_state.fun_ref->ast_root->as()->get_body()->as(); + bool is_last_statement = body_block->get_item(body_block->size() - 1) == v; + cur_state.has_returns_in_the_middle |= !is_last_statement; + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // unsupported or no-sense cases + if (fun_ref->is_builtin_function() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || + fun_ref->has_tvm_method_id() || !fun_ref->arg_order.empty() || !fun_ref->ret_order.empty() || + fun_ref->is_used_as_noncall()) { + return false; + } + // disabled by the user + if (fun_ref->inline_mode == FunctionInlineMode::noInline || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { + return false; + } + // okay, start auto-detection; + // for functions marked `@inline` (inlineViaFif), probably we'll change to inlineInPlace + return true; + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_state = StateWhileTraversingFunction(fun_ref); + collected_expect_inline.clear(); + parent::visit(v_function->get_body()); + + bool prevented_anyway = cur_state.is_inlining_prevented_even_if_annotated(); + bool will_inline = false; + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { + // if a function is marked `@inline`, so the user requested in to be inlined; + // if it's possible, do it; otherwise, leave it as `PROCINLINE` to Fift + will_inline = !prevented_anyway; + } else { + // a function is not marked `@inline` / `@inline_ref` / etc., so automatically decide + will_inline = !prevented_anyway && cur_state.should_auto_inline_if_not_prevented(); + } + + // handle `__expect_inline(true)` (assertions inside compiler tests) + for (auto v_expect : collected_expect_inline) { + tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); + bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; + if (expected != will_inline) { + fire(fun_ref, v_expect->loc, "__expect_inline failed"); + } + } + + // okay, this function will be inlined, mark the flag + bool is_called = fun_ref->n_times_called || is_called_implicitly_by_compiler(fun_ref); + if (will_inline && is_called) { + fun_ref->mutate()->assign_inline_mode_in_place(); + } + } +}; + +// this visitor (called once for a function): +// 1) fills call_graph[cur_f] (all function calls from cur_f) +// 2) increments n_times_called +// as a result of applying it to every function, we get a full call graph and how many times each function was called; +// we'll use this call graph to detect recursive components (functions within recursions can not be inlined) +class CallGraphBuilderVisitor final : ASTVisitorFunctionBody { + FunctionPtr cur_f{nullptr}; + +protected: + void visit(V v) override { + if (FunctionPtr called_f = v->fun_maybe) { + if (called_f->is_code_function()) { + call_graph[cur_f].emplace_back(called_f); + } + called_f->mutate()->n_times_called++; + } + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // don't include asm functions, we don't need them in calculations + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + call_graph[cur_f] = std::vector{}; + parent::visit(v_function->get_body()); + } +}; + +static void detect_recursive_functions() { + // 1) build call_graph (and calculate n_times_called also) + visit_ast_of_all_functions(); + + // 2) using call_graph, detect cycles (the smallest, non-optimized algorithm, okay for our needs) + for (const auto& it : call_graph) { + FunctionPtr f_start_from = it.first; + std::unordered_set visited; + std::function is_recursive_dfs = [&](FunctionPtr cur) -> bool { + for (FunctionPtr f_called : call_graph[cur]) { + if (f_called == f_start_from) + return true; + if (!visited.insert(f_called).second) + continue; + if (is_recursive_dfs(f_called)) + return true; + } + return false; + }; + if (!it.second.empty() && is_recursive_dfs(f_start_from)) { + f_start_from->mutate()->n_times_called = 9999; // means "recursive" + } + } +} + +void pipeline_detect_inline_in_place() { + detect_recursive_functions(); + visit_ast_of_all_functions(); + call_graph.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index d31348ba9..1cadf07f4 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -25,7 +25,6 @@ */ #include "tolk.h" #include "ast.h" -#include "ast-from-tokens.h" #include "compiler-state.h" /* @@ -38,6 +37,8 @@ namespace tolk { +AnyV parse_src_file_to_ast(const SrcFile* file); + void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); @@ -51,10 +52,9 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { if (auto v_import = v_toplevel->try_as()) { std::string imported_str = v_import->get_file_name(); - size_t cur_slash_pos = file->rel_filename.rfind('/'); - std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@' + std::string rel_filename = imported_str[0] == '@' ? std::move(imported_str) - : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str; + : file->extract_dirname() + imported_str; const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); file->imports.push_back(SrcFile::ImportDirective{imported}); @@ -64,7 +64,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con } // todo #ifdef TOLK_PROFILING - lexer_measure_performance(G.all_src_files); + // lexer_measure_performance(G.all_src_files); } } // namespace tolk diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp index 2b7e55578..dad565e0e 100644 --- a/tolk/pipe-find-unused-symbols.cpp +++ b/tolk/pipe-find-unused-symbols.cpp @@ -37,7 +37,7 @@ namespace tolk { static void mark_function_used_dfs(const std::unique_ptr& op); static void mark_function_used(FunctionPtr fun_ref) { - if (!fun_ref->is_code_function() || fun_ref->is_really_used()) { // already handled + if (!fun_ref->is_code_function() || fun_ref->is_really_used() || fun_ref->is_inlined_in_place()) { // already handled return; } @@ -67,7 +67,7 @@ static void mark_function_used_dfs(const std::unique_ptr& op) { void pipeline_find_unused_symbols() { for (FunctionPtr fun_ref : G.all_functions) { - if (fun_ref->is_method_id_not_empty()) { // get methods, main and other entrypoints, regular functions with @method_id + if (fun_ref->has_tvm_method_id()) { // get methods, main and other entrypoints, regular functions with @method_id mark_function_used(fun_ref); } } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 57f481f07..1dc513f05 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -47,7 +47,7 @@ static void generate_output_func(FunctionPtr fun_ref) { CodeBlob* code = std::get(fun_ref->body)->code; if (G.is_verbosity(3)) { - code->print(std::cerr, 9); + code->print(std::cerr, 0); } code->prune_unreachable_code(); if (G.is_verbosity(5)) { @@ -73,30 +73,48 @@ static void generate_output_func(FunctionPtr fun_ref) { } code->mark_noreturn(); if (G.is_verbosity(3)) { - code->print(std::cerr, 15); + // code->print(std::cerr, 15); } if (G.is_verbosity(2)) { std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; } const char* modifier = ""; - if (fun_ref->is_inline()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { modifier = "INLINE"; - } else if (fun_ref->is_inline_ref()) { + } else if (fun_ref->inline_mode == FunctionInlineMode::inlineRef) { modifier = "REF"; } - std::cout << std::string(2, ' ') << fun_ref->name << " PROC" << modifier << ":<{\n"; + if (G.settings.tolk_src_as_line_comments) { + std::cout << " // " << fun_ref->loc; + if (!fun_ref->n_times_called && !fun_ref->is_used_as_noncall() && !fun_ref->is_entrypoint() && !fun_ref->has_tvm_method_id()) { + std::cout << " (note: function never called!)"; + } + std::cout << std::endl; + } + std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; if (G.settings.stack_layout_comments) { - mode |= Stack::_StkCmt | Stack::_CptStkCmt; - } - if (fun_ref->is_inline() && code->ops->noreturn()) { + mode |= Stack::_StackComments; + size_t len = 2 + fun_ref->name.size() + 5 + std::strlen(modifier) + 3; + while (len < 28) { // a bit weird, but okay for now: + std::cout << ' '; // insert space after "xxx() PROC" before `// stack state` + len++; // (the first AsmOp-comment that will be code generated) + } // space is the same as used to align comments in asmops.cpp + std::cout << '\t'; + } else { + std::cout << std::endl; + } + if (G.settings.tolk_src_as_line_comments) { + mode |= Stack::_LineComments; + } + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif && code->ops->noreturn()) { mode |= Stack::_InlineFunc; } - if (fun_ref->is_inline() || fun_ref->is_inline_ref()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { mode |= Stack::_InlineAny; } code->generate_code(std::cout, mode, 2); - std::cout << std::string(2, ' ') << "}>\n"; + std::cout << " " << "}>\n"; if (G.is_verbosity(2)) { std::cerr << "--------------\n"; } @@ -107,11 +125,11 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "// automatically generated from "; bool need_comma = false; for (const SrcFile* file : G.all_src_files) { - if (!file->is_stdlib_file()) { + if (!file->is_stdlib_file) { if (need_comma) { std::cout << ", "; } - std::cout << file->rel_filename; + std::cout << file->extract_short_name(); need_comma = true; } } @@ -119,11 +137,13 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "PROGRAM{\n"; bool has_main_procedure = false; + int n_inlined_in_place = 0; for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { if (G.is_verbosity(2) && fun_ref->is_code_function()) { std::cerr << fun_ref->name << ": code not generated, function does not need codegen\n"; } + n_inlined_in_place += fun_ref->is_inlined_in_place(); continue; } @@ -131,11 +151,11 @@ void pipeline_generate_fif_output_to_std_cout() { has_main_procedure = true; } - std::cout << std::string(2, ' '); - if (fun_ref->is_method_id_not_empty()) { - std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; + std::cout << " "; + if (fun_ref->has_tvm_method_id()) { + std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "()\n"; } else { - std::cout << "DECLPROC " << fun_ref->name << "\n"; + std::cout << "DECLPROC " << fun_ref->name << "()\n"; } } @@ -143,6 +163,15 @@ void pipeline_generate_fif_output_to_std_cout() { throw Fatal("the contract has no entrypoint; forgot `fun onInternalMessage(...)`?"); } + if (n_inlined_in_place) { + std::cout << " // " << n_inlined_in_place << " functions inlined in-place:" << "\n"; + for (FunctionPtr fun_ref : G.all_functions) { + if (fun_ref->is_inlined_in_place()) { + std::cout << " // - " << fun_ref->name << " (" << fun_ref->n_times_called << (fun_ref->n_times_called == 1 ? " call" : " calls") << ")\n"; + } + } + } + for (GlobalVarPtr var_ref : G.all_global_vars) { if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { if (G.is_verbosity(2)) { @@ -151,11 +180,11 @@ void pipeline_generate_fif_output_to_std_cout() { continue; } - std::cout << std::string(2, ' ') << "DECLGLOBVAR " << var_ref->name << "\n"; + std::cout << " " << "DECLGLOBVAR $" << var_ref->name << "\n"; } for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { continue; } generate_output_func(fun_ref); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 5fb12059f..1bca8908a 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -21,6 +21,7 @@ #include "generics-helpers.h" #include "type-system.h" #include "smart-casts-cfg.h" +#include /* * This is a complicated and crucial part of the pipeline. It simultaneously does the following: @@ -96,6 +97,7 @@ namespace tolk { static void infer_and_save_return_type_of_function(FunctionPtr fun_ref); +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref); static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { if (!fun_ref->inferred_return_type) { @@ -119,27 +121,160 @@ static std::string to_string(FunctionPtr fun_ref) { return "`" + fun_ref->as_human_readable() + "`"; } -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(StructPtr struct_ref) { + return "`" + struct_ref->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AliasDefPtr alias_ref) { + return "`" + alias_ref->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); } // fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const std::vector& substitutions, int arg_idx) { - fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + fun_ref->genericTs->get_nameT(arg_idx) + "=" + substitutions[arg_idx]->as_human_readable() + ", because it occupies " + std::to_string(substitutions[arg_idx]->get_width_on_stack()) + " stack slots in TVM, not 1"); +static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { + fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); } // fire an error on `untypedTupleVar.0` when used without a hint GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { std::string idx_access = "." + std::to_string(index); - fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); + fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); +} + +// fire an error on using lateinit variable before definite assignment +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcLocation loc, std::string_view name) { + fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); +} + +// fire an error when `obj.f()`, method `f` not found, try to locate a method for another type +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view method_name) { + if (std::vector other = lookup_methods_with_name(method_name); !other.empty()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); + } + if (const Symbol* sym = lookup_global_symbol(method_name); sym && sym->try_as()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found, but there is a global function named `" + to_string(method_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + } + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); +} + +// safe version of std::stoi that does not crash on long numbers +static bool try_parse_string_to_int(std::string_view str, int& out) { + auto result = std::from_chars(str.data(), str.data() + str.size(), out); + return result.ec == std::errc() && result.ptr == str.data() + str.size(); +} + +// helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` +// example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { + // example: `var w: Ok = Ok { ... }`, hint is `Ok`, lookup is `Ok` + if (const TypeDataStruct* h_struct = hint->unwrap_alias()->try_as()) { + if (lookup_ref == h_struct->struct_ref->base_struct_ref) { + return h_struct; + } + } + // example: `fun f(): Response { return Err { ... } }`, hint is `Ok | Err`, lookup is `Err` + if (const TypeDataUnion* h_union = hint->unwrap_alias()->try_as()) { + TypePtr only_variant = nullptr; // hint `Ok | Ok` is ambiguous + for (TypePtr h_variant : h_union->variants) { + if (const TypeDataStruct* variant_struct = h_variant->unwrap_alias()->try_as()) { + if (lookup_ref == variant_struct->struct_ref->base_struct_ref) { + if (only_variant) { + return nullptr; + } + only_variant = variant_struct; + } + } + } + return only_variant; + } + return nullptr; +} + +// helper function, similar to the above, but for generic type aliases +// example: `v is OkAlias`, need to deduce `OkAlias` based on type of v +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr lookup_ref) { + // when a generic type alias points to a generic struct actually: `type WrapperAlias = Wrapper` + if (const TypeDataGenericTypeWithTs* as_instT = lookup_ref->underlying_type->try_as()) { + return as_instT->struct_ref + ? try_pick_instantiated_generic_from_hint(hint, as_instT->struct_ref) + : try_pick_instantiated_generic_from_hint(hint, as_instT->alias_ref); + } + // it's something weird, when a generic alias refs non-generic type + // example: `type StrangeInt = int`, hint is `StrangeInt`, lookup `StrangeInt` + if (const TypeDataAlias* h_alias = hint->try_as()) { + if (lookup_ref == h_alias->alias_ref->base_alias_ref) { + return h_alias; + } + } + return nullptr; +} + +// given `p.create` (called_receiver = Point, called_name = "create") +// look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) +static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { + // most practical cases: `builder.storeInt` etc., when a direct method for receiver exists + if (FunctionPtr exact_method = match_exact_method_for_call_not_generic(called_receiver, called_name)) { + return {exact_method, GenericsSubstitutions(exact_method->genericTs)}; + } + + // else, try to match, for example `10.copy` with `int?.copy`, with `T.copy`, etc. + std::vector candidates = match_methods_for_call_including_generic(called_receiver, called_name); + if (candidates.size() == 1) { + return candidates[0]; + } + if (candidates.empty()) { // return nullptr, the caller side decides how to react on this + return {nullptr, GenericsSubstitutions(nullptr)}; + } + + std::ostringstream msg; + msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; + for (const auto& [method_ref, substitutedTs] : candidates) { + msg << "candidate function: `" << method_ref->as_human_readable() << "`"; + if (method_ref->is_generic_function()) { + msg << " with " << substitutedTs.as_human_readable(false); + } + if (method_ref->loc.is_defined()) { + msg << " (declared at " << method_ref->loc << ")\n"; + } else if (method_ref->is_builtin_function()) { + msg << " (builtin)\n"; + } + } + fire(cur_f, loc, msg.str()); } +// given fun `f` and a call `f(a,b,c)`, check that argument count is expected; +// (parameters may have default values, so it's not as trivial as to compare params and args size) +void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v, FunctionPtr called_f, AnyExprV self_obj) { + int delta_self = self_obj != nullptr; + int n_arguments = v->get_num_args() + delta_self; + int n_max_params = called_f->get_num_params(); + int n_min_params = n_max_params; + while (n_min_params && called_f->get_param(n_min_params - 1).has_default_value()) { + n_min_params--; + } + + if (!called_f->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` + fire(cur_f, v->loc, "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); + } + if (n_max_params < n_arguments) { + fire(cur_f, v->loc, "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + if (n_arguments < n_min_params) { + fire(cur_f, v->loc, "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } +} /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. @@ -155,7 +290,7 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, AnyExprV src) { #ifdef TOLK_DEBUG - tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_unresolved_inside() && !src->inferred_type->has_genericT_inside()); + tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(src->inferred_type); } @@ -163,30 +298,30 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(LocalVarPtr local_var_or_param, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif local_var_or_param->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(FunctionPtr fun_ref, TypePtr inferred_return_type, TypePtr inferred_full_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_unresolved_inside() && !inferred_return_type->has_genericT_inside()); + tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_genericT_inside()); #endif fun_ref->mutate()->assign_inferred_type(inferred_return_type, inferred_full_type); } // traverse children in any statement FlowContext process_any_statement(AnyV v, FlowContext&& flow) { - switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), std::move(flow)); + switch (v->kind) { + case ast_block_statement: + return process_block_statement(v->as(), std::move(flow)); case ast_return_statement: return process_return_statement(v->as(), std::move(flow)); case ast_if_statement: @@ -214,7 +349,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // returns ExprFlow: out_facts that are "definitely known" after evaluating the whole expression // if used_as_condition, true_facts/false_facts are also calculated (don't calculate them always for optimization) ExprFlow infer_any_expr(AnyExprV v, FlowContext&& flow, bool used_as_condition, TypePtr hint = nullptr) { - switch (v->type) { + switch (v->kind) { case ast_int_const: return infer_int_const(v->as(), std::move(flow), used_as_condition); case ast_string_const: @@ -237,30 +372,38 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_ternary_operator(v->as(), std::move(flow), used_as_condition, hint); case ast_cast_as_operator: return infer_cast_as_operator(v->as(), std::move(flow), used_as_condition); + case ast_is_type_operator: + return infer_is_type_operator(v->as(), std::move(flow), used_as_condition); case ast_not_null_operator: return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); - case ast_is_null_check: - return infer_is_null_check(v->as(), std::move(flow), used_as_condition); + case ast_lazy_operator: + return infer_lazy_operator(v->as(), std::move(flow), used_as_condition); case ast_parenthesized_expression: return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); + case ast_braced_expression: + return infer_braced_expression(v->as(), std::move(flow), used_as_condition, hint); case ast_reference: - return infer_reference(v->as(), std::move(flow), used_as_condition); + return infer_reference(v->as(), std::move(flow), used_as_condition, hint); case ast_dot_access: return infer_dot_access(v->as(), std::move(flow), used_as_condition, hint); case ast_function_call: return infer_function_call(v->as(), std::move(flow), used_as_condition, hint); case ast_tensor: return infer_tensor(v->as(), std::move(flow), used_as_condition, hint); - case ast_typed_tuple: - return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); + case ast_bracket_tuple: + return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); case ast_null_keyword: return infer_null_keyword(v->as(), std::move(flow), used_as_condition); + case ast_match_expression: + return infer_match_expression(v->as(), std::move(flow), used_as_condition, hint); + case ast_object_literal: + return infer_object_literal(v->as(), std::move(flow), used_as_condition, hint); case ast_underscore: return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); case ast_empty_expression: return infer_empty_expression(v->as(), std::move(flow), used_as_condition); default: - throw UnexpectedASTNodeType(v, "infer_any_expr"); + throw UnexpectedASTNodeKind(v, "infer_any_expr"); } } @@ -279,12 +422,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } static ExprFlow infer_string_const(V v, FlowContext&& flow, bool used_as_condition) { - if (v->is_bitslice()) { - assign_inferred_type(v, TypeDataSlice::create()); - } else { - assign_inferred_type(v, TypeDataInt::create()); - } - + assign_inferred_type(v, TypeDataSlice::create()); return ExprFlow(std::move(flow), used_as_condition); } @@ -316,7 +454,8 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->marked_as_redef) { assign_inferred_type(v, v->var_ref->declared_type); } else { - assign_inferred_type(v, v->declared_type ? v->declared_type : TypeDataUnknown::create()); + assign_inferred_type(v, v->type_node ? v->type_node->resolved_type : TypeDataUnknown::create()); + flow.register_known_type(SinkExpression(v->var_ref), TypeDataUnknown::create()); // it's unknown before assigned } return ExprFlow(std::move(flow), used_as_condition); } @@ -350,14 +489,14 @@ class InferTypesAndCallsAndFieldsVisitor final { } assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); - } else if (auto lhs_tuple = lhs->try_as()) { + } else if (auto lhs_tuple = lhs->try_as()) { std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { flow = infer_left_side_of_assignment(lhs_tuple->get_item(i), std::move(flow)); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); } else if (auto lhs_par = lhs->try_as()) { flow = infer_left_side_of_assignment(lhs_par->get_expr(), std::move(flow)); @@ -392,12 +531,12 @@ class InferTypesAndCallsAndFieldsVisitor final { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->declared_type; + TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->inferred_type == TypeDataUnknown::create()) { assign_inferred_type(lhs_var, rhs_type); assign_inferred_type(lhs_var->var_ref, rhs_type); } - TypePtr smart_casted_type = declared_type ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; + TypePtr smart_casted_type = declared_type && rhs_type != TypeDataUnknown::create() ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; out_flow.register_known_type(SinkExpression(lhs_var->var_ref), smart_casted_type); return; } @@ -405,7 +544,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tensor->size()); for (int i = 0; i < lhs_tensor->size(); ++i) { @@ -419,8 +558,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -428,7 +567,7 @@ class InferTypesAndCallsAndFieldsVisitor final { process_assignment_lhs_after_infer_rhs(lhs_tuple->get_item(i), ith_rhs_type, out_flow); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); return; } @@ -457,14 +596,13 @@ class InferTypesAndCallsAndFieldsVisitor final { FlowContext rhs_flow = std::move(after_lhs.out_flow); ExprFlow after_rhs = infer_any_expr(rhs, std::move(rhs_flow), false, lhs->inferred_type); - // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + // all operators implementation is hardcoded by built-in functions `_+_` and similar std::string_view builtin_func = v->operator_name; // "+" for operator += assign_inferred_type(v, lhs); - if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); - v->mutate()->assign_fun_ref(builtin_sym); - } + + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); + v->mutate()->assign_fun_ref(builtin_sym); return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); } @@ -483,7 +621,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, TypeDataInt::create()); break; case tok_logical_not: - if (rhs->inferred_type == TypeDataBool::create()) { + if (rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { builtin_func = "!b"; // "overloaded" for bool } assign_inferred_type(v, TypeDataBool::create()); @@ -493,7 +631,7 @@ class InferTypesAndCallsAndFieldsVisitor final { tolk_assert(false); } - FunctionPtr builtin_sym = lookup_global_symbol(static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function(static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); return after_rhs; @@ -525,12 +663,11 @@ class InferTypesAndCallsAndFieldsVisitor final { case tok_bitwise_xor: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - if (lhs->inferred_type == TypeDataBool::create() && rhs->inferred_type == TypeDataBool::create()) { + if (lhs->inferred_type->unwrap_alias() == TypeDataBool::create() && rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { assign_inferred_type(v, TypeDataBool::create()); } else { assign_inferred_type(v, TypeDataInt::create()); } - assign_inferred_type(v, rhs); // (int & int) is int, (bool & bool) is bool break; // && || result in booleans, but building flow facts is tricky due to short-circuit case tok_logical_and: { @@ -560,14 +697,19 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - assign_inferred_type(v, TypeDataInt::create()); + if ((v->tok == tok_plus || v->tok == tok_minus) && lhs->inferred_type == TypeDataCoins::create()) { + assign_inferred_type(v, TypeDataCoins::create()); // coins + coins = coins + } else { + assign_inferred_type(v, TypeDataInt::create()); // int8 + int8 = int, as well as other operators/types + } } if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); } @@ -590,12 +732,18 @@ class InferTypesAndCallsAndFieldsVisitor final { return after_false; } - TypeInferringUnifyStrategy tern_type; - tern_type.unify_with(v->get_when_true()->inferred_type); - if (!tern_type.unify_with(v->get_when_false()->inferred_type)) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + TypeInferringUnifyStrategy branches_unifier; + branches_unifier.unify_with(v->get_when_true()->inferred_type, hint); + branches_unifier.unify_with(v->get_when_false()->inferred_type, hint); + if (branches_unifier.is_union_of_different_types()) { + // `... ? intVar : sliceVar` results in `int | slice`, probably it's not what the user expected + // example: `var v = ternary`, show an inference error + // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); + } } - assign_inferred_type(v, tern_type.get_result()); + assign_inferred_type(v, branches_unifier.get_result()); FlowContext out_flow = FlowContext::merge_flow(std::move(after_true.out_flow), std::move(after_false.out_flow)); return ExprFlow(std::move(out_flow), std::move(after_true.true_flow), std::move(after_false.false_flow)); @@ -603,8 +751,8 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_cast_as_operator(V v, FlowContext&& flow, bool used_as_condition) { // for `expr as `, use this type for hint, so that `t.tupleAt(0) as int` is ok - ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->cast_to_type); - assign_inferred_type(v, v->cast_to_type); + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->type_node->resolved_type); + assign_inferred_type(v, v->type_node->resolved_type); if (!used_as_condition) { return after_expr; @@ -612,15 +760,37 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } - ExprFlow infer_is_null_check(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_is_type_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr expr_type = v->get_expr()->inferred_type; - TypePtr non_null_type = calculate_type_subtract_null(expr_type); - if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true + TypePtr rhs_type = v->type_node->resolved_type; + + if (const auto* t_struct = rhs_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `v is Wrapper`, detect T based on type of v (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_struct->struct_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = rhs_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `v is WrapperAlias`, detect T similar to structures + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_alias->alias_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + + rhs_type = rhs_type->unwrap_alias(); + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); + if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); - } else if (non_null_type == TypeDataNever::create()) { // `expr == null` is always false + } else if (non_rhs_type == TypeDataNever::create()) { // `expr is ` is always false v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); } else { v->mutate()->assign_always_true_or_false(0); @@ -640,11 +810,11 @@ class InferTypesAndCallsAndFieldsVisitor final { true_flow.mark_unreachable(UnreachableKind::CantHappen); true_flow.register_known_type(s_expr, TypeDataNever::create()); } else if (!v->is_negated) { - true_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); - false_flow.register_known_type(s_expr, non_null_type); + true_flow.register_known_type(s_expr, rhs_type); + false_flow.register_known_type(s_expr, non_rhs_type); } else { - true_flow.register_known_type(s_expr, non_null_type); - false_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + true_flow.register_known_type(s_expr, non_rhs_type); + false_flow.register_known_type(s_expr, rhs_type); } } return ExprFlow(std::move(after_expr.out_flow), std::move(true_flow), std::move(false_flow)); @@ -652,12 +822,9 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); - - if (const auto* as_nullable = v->get_expr()->inferred_type->try_as()) { - assign_inferred_type(v, as_nullable->inner); - } else { - assign_inferred_type(v, v->get_expr()); - } + TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type->unwrap_alias(), TypeDataNullLiteral::create()); + assign_inferred_type(v, without_null_type != TypeDataNever::create() ? without_null_type : expr_type); if (!used_as_condition) { return after_expr; @@ -665,47 +832,134 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } + ExprFlow infer_lazy_operator(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow lazy_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition); + assign_inferred_type(v, v->get_expr()); // there is no Lazy, so `lazy expr` is just typeof expr + return lazy_expr; + } + ExprFlow infer_parenthesized(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition, hint); assign_inferred_type(v, v->get_expr()); return after_expr; } - ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex (used to represent a non-braced `match` arm) + AnyExprV implicit_result = nullptr; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_result == nullptr); + implicit_result = v_return; + flow = infer_any_expr(v_return->get_expr(), std::move(flow), false, hint).out_flow; + assign_inferred_type(v_return, v_return->get_expr()); + } else { + flow = process_any_statement(item, std::move(flow)); + } + } + + assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : implicit_result ? implicit_result->inferred_type : TypeDataVoid::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + + // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), + // validate and collect them + // returns: [int, slice] / [cell] + std::vector collect_type_arguments_for_fun(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + // for `genericF` user should provide two Ts + // for `Container.wrap` — one U (and one T is implicitly from receiver) + if (instantiationT_list->size() != genericTs->size() - genericTs->n_from_receiver) { + fire(cur_f, loc, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); + } + + std::vector type_arguments; + type_arguments.reserve(instantiationT_list->size()); + for (int i = 0; i < instantiationT_list->size(); ++i) { + type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); + } + + return type_arguments; + } + + // when substitutedTs have been collected from `f` or deduced from arguments, instantiate a generic function `f` + // example: was `t.push(2)`, deduced , instantiate `tuple.push` + // example: was `t.push(2)`, collected , instantiate `tuple.push` (will later fail type check) + // example: was `var cb = t.first;` (used as reference, as non-call), instantiate `tuple.first` + // returns fun_ref to instantiated function + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { + // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly + if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + for (int i = 0; i < substitutedTs.size(); ++i) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !fun_ref->is_variadic_width_T_allowed()) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + } + } + } + + // make deep clone of `f` with substitutedTs (or immediately return from symtable if already instantiated) + return instantiate_generic_function(fun_ref, std::move(substitutedTs)); + } + + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr) const { + // at current point, v is a reference: + // - either a standalone: `local_var` / `SOME_CONST` / `globalF` / `genericFn` + // - or inside a call: `globalF()` / `genericFn()` / `genericFn()` / `local_var()` + if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow assign_inferred_type(v, declared_or_smart_casted); + if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { + fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); + } + // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression } else if (GlobalConstPtr const_ref = v->sym->try_as()) { - assign_inferred_type(v, const_ref->is_int_const() ? TypeDataInt::create() : TypeDataSlice::create()); + if (!const_ref->inferred_type) { + infer_and_save_type_of_constant(const_ref); + } + assign_inferred_type(v, const_ref->inferred_type); } else if (GlobalVarPtr glob_ref = v->sym->try_as()) { // there are no smart casts for globals, it's a way of preventing reading one global multiple times, it costs gas assign_inferred_type(v, glob_ref->declared_type); } else if (FunctionPtr fun_ref = v->sym->try_as()) { - // it's `globalF` / `globalF` - references to functions used as non-call + // it's `globalF` / `globalF` / `globalF()` / `globalF()` + // if it's a call, then out_f_called is present, we should fill it V v_instantiationTs = v->get_instantiationTs(); - if (fun_ref->is_generic_function() && !v_instantiationTs) { + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `genericFn` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `genericFn` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `genericFn` is valid as non-call, it's a reference to instantiation + // `genericFn()` is also ok, we'll assign an instantiated fun_ref to out_f_called + GenericsSubstitutions substitutedTs(fun_ref->genericTs); + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); - } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { + } else if (UNLIKELY(v_instantiationTs != nullptr)) { // non-generic function referenced like `return beginCell;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); + if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + } else { // so, it's `globalF` / `genericFn` as a reference + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { + fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); + } + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, "can not get reference to this function, it's a special entrypoint"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); } else { @@ -714,65 +968,64 @@ class InferTypesAndCallsAndFieldsVisitor final { // for non-functions: `local_var` and similar not allowed if (UNLIKELY(v->has_instantiationTs())) { - fire(cur_f, v->get_instantiationTs()->loc, "generic T not expected here"); + fire(cur_f, v->get_instantiationTs()->loc, "type arguments not expected here"); } return ExprFlow(std::move(flow), used_as_condition); } - // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), - // validate and collect them - // returns: [int, slice] / [cell] - std::vector collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, FunctionPtr fun_ref, V instantiationT_list) const { - if (fun_ref->genericTs->size() != instantiationT_list->get_items().size()) { - fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(fun_ref->genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); - } - - std::vector substitutions; - substitutions.reserve(instantiationT_list->size()); - for (int i = 0; i < instantiationT_list->size(); ++i) { - substitutions.push_back(instantiationT_list->get_item(i)->substituted_type); - } + ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr, AnyExprV* out_dot_obj = nullptr) { + // at current point, v is a dot access to a field / index / method: + // - either a standalone: `user.id` / `getUser().id` / `var.0` / `t.size` / `Point.create` / `t.tupleAt` + // - or inside a call: `user.getId()` / `.method()` / `Point.create()` / `t.tupleAt(1)` - return substitutions; - } + AnyExprV dot_obj = nullptr; // to be filled for `.(field/index/method)`, nullptr for `Point.create` + FunctionPtr fun_ref = nullptr; // to be filled for `.method` / `Point.create` (both standalone or in a call) + GenericsSubstitutions substitutedTs(nullptr); + V v_ident = v->get_identifier(); // field/method name vertex - // when generic Ts have been collected from user-specified or deduced from arguments, - // instantiate a generic function - // example: was `t.tuplePush(2)`, deduced , instantiate `tuplePush` - // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) - // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` - // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) const { - // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly - if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i]->get_width_on_stack() != 1) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutionTs, i); + // handle `Point.create` / `Container.wrap`: no dot_obj expression actually, lhs is a type, looking up a method + if (auto obj_ref = v->get_obj()->try_as()) { + if (const auto* obj_as_type = obj_ref->sym->try_as()) { + TypePtr receiver_type = obj_as_type->resolved_type; + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + if (!fun_ref) { + fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(receiver_type)); } } } + // handle other (most, actually) cases: `.field` / `.method` + if (!fun_ref) { + dot_obj = v->get_obj(); + flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; + } - std::string inst_name = generate_instantiated_name(fun_ref->name, substitutionTs); - // make deep clone of `f` with substitutionTs - // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(loc, fun_ref, inst_name, std::move(substitutionTs)); - } - - ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - // it's NOT a method call `t.tupleSize()` (since such cases are handled by infer_function_call) - // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) - flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; - - TypePtr obj_type = v->get_obj()->inferred_type; - // our goal is to fill v->target knowing type of obj - V v_ident = v->get_identifier(); // field/method name vertex + // our goal is to fill v->target (field/index/method) knowing type of obj + TypePtr obj_type = dot_obj ? dot_obj->inferred_type->unwrap_alias() : TypeDataUnknown::create(); V v_instantiationTs = v->get_instantiationTs(); std::string_view field_name = v_ident->name; - // it can be indexed access (`tensorVar.0`, `tupleVar.1`) or a method (`t.tupleSize`) - // at first, check for indexed access - if (field_name[0] >= '0' && field_name[0] <= '9') { - int index_at = std::stoi(std::string(field_name)); + // check for field access (`user.id`), when obj is a struct + if (const TypeDataStruct* obj_struct = obj_type->try_as()) { + if (StructFieldPtr field_ref = obj_struct->struct_ref->find_field(field_name)) { + v->mutate()->assign_target(field_ref); + TypePtr inferred_type = field_ref->declared_type; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + // if field_name doesn't exist, don't fire an error now — maybe, it's `user.method()` + } + + // check for indexed access (`tensorVar.0` / `tupleVar.1`) + if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { + int index_at; + if (!try_parse_string_to_int(field_name, index_at)) { + fire(cur_f, v_ident->loc, "invalid numeric index"); + } if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); @@ -787,7 +1040,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = obj_type->try_as()) { + if (const auto* t_tuple = obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -806,7 +1059,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->is_lvalue && !hint) { // left side of assignment item_type = TypeDataUnknown::create(); } else { - if (hint == nullptr) { + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); } item_type = hint; @@ -815,39 +1068,62 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, item_type); return ExprFlow(std::move(flow), used_as_condition); } - fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); } - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists + // for `T.copy` / `Container.create`, substitution for T is also returned if (!fun_ref) { - fire(cur_f, v_ident->loc, "non-existing field `" + static_cast(field_name) + "` of type " + to_string(obj_type)); + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, dot_obj->inferred_type, field_name); } - // `t.tupleSize` is ok, `cs.tupleSize` not - if (!fun_ref->parameters[0].declared_type->can_rhs_be_assigned(obj_type)) { - fire(cur_f, v_ident->loc, "referencing a method for " + to_string(fun_ref->parameters[0].declared_type) + " with object of type " + to_string(obj_type)); + // not a field, not a method — fire an error + if (!fun_ref) { + // as a special case, handle accessing fields of nullable objects, to show a more precise error + if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { + if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); + } + } + if (out_f_called) { + fire_error_method_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, v->get_field_name()); + } else { + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); + } } - if (fun_ref->is_generic_function() && !v_instantiationTs) { - // `genericFn` and `t.tupleAt` are invalid as non-call, they can't be used without + // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it + if (fun_ref->is_generic_function() && fun_ref->genericTs->size() == fun_ref->genericTs->n_from_receiver) { + tolk_assert(substitutedTs.typeT_at(0) != nullptr); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + } + + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { + // `t.tupleAt` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `t.tupleAt` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `t.tupleAt` is valid as non-call, it's a reference to instantiation + // `t.tupleAt()` is also ok, we'll assign an instantiated fun_ref to out_f_called + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic method referenced like `var cb = c.cellHash;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + // non-generic method referenced like `var cb = c.hash;` + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - v->mutate()->assign_target(fun_ref); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); // type of `t.tupleSize` is TypeDataFunCallable + if (out_f_called) { // so, it's `user.method()` / `t.tupleAt()` / `t.tupleAt()` / `Point.create` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + *out_dot_obj = dot_obj; + } else { // so, it's `user.method` / `t.tupleAt` as a reference + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { + fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + v->mutate()->assign_target(fun_ref); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); } @@ -855,49 +1131,21 @@ class InferTypesAndCallsAndFieldsVisitor final { AnyExprV callee = v->get_callee(); // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` - int delta_self = 0; - AnyExprV dot_obj = nullptr; + AnyExprV self_obj = nullptr; // for `obj.method()`, obj will be here (but for `Point.create()`, no obj exists) FunctionPtr fun_ref = nullptr; - V v_instantiationTs = nullptr; - - if (auto v_ref = callee->try_as()) { - // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` - fun_ref = v_ref->sym->try_as(); // not null for `globalF` - v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` - - } else if (auto v_dot = callee->try_as()) { - // `obj.someMethod()` / `obj.someMethod()` / `getF().someMethod()` / `obj.SOME_CONST()` - // note, that dot_obj->target is not filled yet, since callee was not inferred yet - delta_self = 1; - dot_obj = v_dot->get_obj(); - v_instantiationTs = v_dot->get_instantiationTs(); // present for `obj.someMethod()` - flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; - - // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) - std::string_view field_name = v_dot->get_field_name(); - if (field_name[0] >= '0' && field_name[0] <= '9') { - // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr - // infer_dot_access() will be called for a callee, it will check index correctness - } else { - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref) { - fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + static_cast(field_name) + "` of type " + to_string(dot_obj)); - } - } + if (callee->kind == ast_reference) { + flow = infer_reference(callee->as(), std::move(flow), false, nullptr, &fun_ref).out_flow; + } else if (callee->kind == ast_dot_access) { + flow = infer_dot_access(callee->as(), std::move(flow), false, nullptr, &fun_ref, &self_obj).out_flow; } else { - // `getF()()` / `5()` - // fun_ref remains nullptr + flow = infer_any_expr(callee, std::move(flow), false).out_flow; } // handle `local_var()` / `getF()()` / `5()` / `SOME_CONST()` / `obj.method()()()` / `tensorVar.0()` if (!fun_ref) { - // treat callee like a usual expression - flow = infer_any_expr(callee, std::move(flow), false).out_flow; - // it must have "callable" inferred type - const TypeDataFunCallable* f_callable = callee->inferred_type->try_as(); + // callee must have "callable" inferred type + const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); } @@ -910,69 +1158,61 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(arg_i, std::move(flow), false, f_callable->params_types[i]).out_flow; assign_inferred_type(v->get_arg(i), arg_i); } - v->mutate()->assign_fun_ref(nullptr); // no fun_ref to a global function assign_inferred_type(v, f_callable->return_type); return ExprFlow(std::move(flow), used_as_condition); } - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - // we're going to iterate over passed arguments, and (if generic) infer substitutionTs - // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) - int n_arguments = v->get_num_args() + delta_self; - int n_parameters = fun_ref->get_num_params(); - if (!n_parameters && dot_obj) { - fire(cur_f, v->loc, "`" + fun_ref->name + "` has no parameters and can not be called as method"); - } - if (n_parameters < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); - } - if (n_arguments < n_parameters) { - fire(cur_f, v->loc, "too few arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + // prevent calling `onBouncedMessage()` and other special functions directly + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); } - // now, for every passed argument, we need to infer its type - // for regular functions, it's obvious - // but for generic functions, we need to infer type arguments (substitutionTs) on the fly - // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) - GenericSubstitutionsDeduceForCall* deducingTs = fun_ref->is_generic_function() ? new GenericSubstitutionsDeduceForCall(fun_ref) : nullptr; - if (deducingTs && v_instantiationTs) { - deducingTs->provide_manually_specified(collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs)); - } + // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) + // we're going to iterate over passed arguments, and (if generic) infer substitutedTs + // at first, check argument count + int delta_self = self_obj != nullptr; + check_arguments_count_at_fun_call(cur_f, v, fun_ref, self_obj); - // loop over every argument, for `obj.method()` obj is the first one - // if genericT deducing has a conflict, ParseError is thrown - // note, that deducing Ts one by one is important to manage control flow (mutate params work like assignments) - // a corner case, e.g. `f(v1:T?, v2:T?)` and `f(null,2)` will fail on first argument, won't try the second one - if (dot_obj) { + // for every passed argument, we need to infer its type + // for generic functions, we need to infer type arguments (substitutedTs) on the fly + // (if they are specified by a user like `f(args)` / `t.tupleAt()`, fun_ref is already instantiated) + GenericSubstitutionsDeducing deducingTs(fun_ref); + + // for `obj.method()` obj is the first argument (passed to `self` parameter) + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); } - if (param_0.is_mutate_parameter() && dot_obj->inferred_type != param_type) { - if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { - assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); + if (param_0.is_mutate_parameter() && self_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { + assign_inferred_type(self_obj, calc_declared_type_before_smart_cast(self_obj)); flow.register_known_type(s_expr, param_type); } } } + // for static generic method call `Container.create` of fun_ref = `Container.create`, gather T=int + if (!self_obj && fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside() && callee->kind == ast_dot_access && callee->as()->get_obj()->kind == ast_reference) { + const auto* t_static = callee->as()->get_obj()->as()->sym->try_as(); + tolk_assert(t_static); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->receiver_type, t_static->resolved_type); + } + + // loop over every argument, one by one, like control flow goes for (int i = 0; i < v->get_num_args(); ++i) { const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; AnyExprV arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = param_i.declared_type; - if (param_type->has_genericT_inside() && deducingTs->is_manually_specified()) { // `f(a)` - param_type = deducingTs->replace_by_manually_specified(param_type); + if (param_type->has_genericT_inside()) { // `fun f(a:T, b:T)` T was fixated on `a`, use it as hint for `b` + param_type = deducingTs.replace_Ts_with_currently_deduced(param_type); } + flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type - // then arg_i is inferred without any hint - flow = infer_any_expr(arg_i, std::move(flow), false).out_flow; - param_type = deducingTs->auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); - } else { - // param_type is hint, helps infer arg_i - flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; + param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression - if (param_i.is_mutate_parameter() && arg_i->inferred_type != param_type) { + if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); flow.register_known_type(s_expr, param_type); @@ -985,43 +1225,37 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known - int idx = deducingTs->get_first_not_deduced_idx(); - if (idx != -1 && hint && fun_ref->declared_return_type->has_genericT_inside()) { + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty() && hint && !hint->has_genericT_inside() && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint - deducingTs->auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); - idx = deducingTs->get_first_not_deduced_idx(); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } - if (idx != -1) { - fire(cur_f, v->loc, "can not deduce " + fun_ref->genericTs->get_nameT(idx)); + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs->flush()); - delete deducingTs; - - } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic function/method called with type arguments, like `c.cellHash()` / `beginCell()` - fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); + } + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); } - v->mutate()->assign_fun_ref(fun_ref); - // since for `t.tupleAt()`, infer_dot_access() not called for callee = "t.tupleAt", assign its target here - if (v->is_dot_call()) { - v->get_callee()->as()->mutate()->assign_target(fun_ref); - } // get return type either from user-specified declaration or infer here on demand traversing its body + v->mutate()->assign_fun_ref(fun_ref, self_obj != nullptr); get_or_infer_return_type(fun_ref); - TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type; + TypePtr inferred_type = fun_ref->does_return_self() ? self_obj->inferred_type : fun_ref->inferred_return_type; assign_inferred_type(v, inferred_type); assign_inferred_type(callee, fun_ref->inferred_full_type); if (inferred_type == TypeDataNever::create()) { flow.mark_unreachable(UnreachableKind::CallNeverReturnFunction); } - // note, that mutate params don't affect typing, they are handled when converting to IR return ExprFlow(std::move(flow), used_as_condition); } ExprFlow infer_tensor(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTensor* tensor_hint = hint ? hint->try_as() : nullptr; + const TypeDataTensor* tensor_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1033,8 +1267,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTypedTuple* tuple_hint = hint ? hint->try_as() : nullptr; + ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataBrackets* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1042,7 +1276,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(item, std::move(flow), false, tuple_hint && i < tuple_hint->size() ? tuple_hint->items[i] : nullptr).out_flow; types_list.emplace_back(item->inferred_type); } - assign_inferred_type(v, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(v, TypeDataBrackets::create(std::move(types_list))); return ExprFlow(std::move(flow), used_as_condition); } @@ -1052,6 +1286,193 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + ExprFlow infer_match_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + flow = infer_any_expr(v->get_subject(), std::move(flow), false).out_flow; + SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_subject()); + TypeInferringUnifyStrategy branches_unifier; + FlowContext arms_entry_facts = flow.clone(); + FlowContext match_out_flow; + bool has_expr_arm = false; + bool has_else_arm = false; + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; + if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + if (const auto* t_struct = exact_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `Wrapper => ...`, detect T based on type of subject (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_struct->struct_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = exact_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `WrapperAlias => ...`, detect T similar to structures + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_alias->alias_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + arm_flow.register_known_type(s_expr, exact_type); + + } else if (v_arm->pattern_kind == MatchArmKind::const_expression) { + has_expr_arm = true; + } else if (v_arm->pattern_kind == MatchArmKind::else_branch) { + has_else_arm = true; + } + + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; + match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); + branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); + } + if (v->get_arms_count() == 0) { + match_out_flow = std::move(arms_entry_facts); + } + if (has_expr_arm && !has_else_arm) { + FlowContext else_flow = process_any_statement(createV(v->loc), std::move(arms_entry_facts)); + match_out_flow = FlowContext::merge_flow(std::move(match_out_flow), std::move(else_flow)); + } + + if (v->is_statement()) { + assign_inferred_type(v, TypeDataVoid::create()); + } else { + if (v->get_arms_count() == 0) { // still allow empty `match` statements, for probable codegen + fire(cur_f, v->loc, "empty `match` can't be used as expression"); + } + if (branches_unifier.is_union_of_different_types()) { + // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { + fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); + } + } + assign_inferred_type(v, branches_unifier.get_result()); + } + + return ExprFlow(std::move(match_out_flow), used_as_condition); + } + + ExprFlow infer_object_literal(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // the goal is to detect struct_ref + // either by lhs hint `var u: User = { ... }, or by explicitly provided ref `User { ... }` + StructPtr struct_ref = nullptr; + + // `User { ... }` / `UserAlias { ... }` / `Wrapper { ... }` / `Wrapper { ... }` + if (v->type_node) { + TypePtr provided_type = v->type_node->resolved_type->unwrap_alias(); + if (const TypeDataStruct* hint_struct = provided_type->try_as()) { + struct_ref = hint_struct->struct_ref; // `Wrapper` / `Wrapper` + } else if (const TypeDataGenericTypeWithTs* hint_instTs = provided_type->try_as()) { + struct_ref = hint_instTs->struct_ref; // if `type WAlias = Wrapper`, here `Wrapper` (generic struct) + } + if (!struct_ref) { + fire(cur_f, v->type_node->loc, to_string(v->type_node->resolved_type) + " does not name a struct"); + } + // example: `var v: Ok = Ok { ... }`, now struct_ref is "Ok", take "Ok" from hint + if (struct_ref->is_generic_struct() && hint) { + if (TypePtr inst_explicit_ref = try_pick_instantiated_generic_from_hint(hint, struct_ref)) { + struct_ref = inst_explicit_ref->try_as()->struct_ref; + } + } + } + if (!struct_ref && hint) { + if (const TypeDataStruct* hint_struct = hint->unwrap_alias()->try_as()) { + struct_ref = hint_struct->struct_ref; + } + if (const TypeDataUnion* hint_union = hint->unwrap_alias()->try_as()) { + StructPtr last_struct = nullptr; + int n_structs = 0; + for (TypePtr variant : hint_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + last_struct = variant_struct->struct_ref; + n_structs++; + } + if (const TypeDataGenericTypeWithTs* hint_withTs = variant->unwrap_alias()->try_as()) { + last_struct = hint_withTs->struct_ref; + n_structs++; + } + } + if (n_structs == 1) { + struct_ref = last_struct; + } + } + if (const TypeDataGenericTypeWithTs* hint_withTs = hint->unwrap_alias()->try_as()) { + struct_ref = hint_withTs->struct_ref; + } + } + if (!struct_ref) { + fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + } + + // so, we have struct_ref, so we can check field names and infer values + // if it's a generic struct, we need to deduce Ts by field values, like for a function call + GenericSubstitutionsDeducing deducingTs(struct_ref); + + uint64_t occurred_mask = 0; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto field_i = v->get_body()->get_field(i); + std::string_view field_name = field_i->get_field_name(); + StructFieldPtr field_ref = struct_ref->find_field(field_name); + if (!field_ref) { + fire(cur_f, field_i->loc, "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); + } + field_i->mutate()->assign_field_ref(field_ref); + + if (occurred_mask & (1ULL << field_ref->field_idx)) { + fire(cur_f, field_i->loc, "duplicate field initialization"); + } + occurred_mask |= 1ULL << field_ref->field_idx; + + AnyExprV val_i = field_i->get_init_val(); + TypePtr field_type = field_ref->declared_type; + if (field_type->has_genericT_inside()) { + field_type = deducingTs.replace_Ts_with_currently_deduced(field_type); + } + flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; + if (field_type->has_genericT_inside()) { + deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); + } + assign_inferred_type(field_i, val_i); + } + + // if it's a generic struct `Wrapper`, we need to instantiate it, like `Wrapper` + if (struct_ref->is_generic_struct()) { + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + } + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT_unknown); + } + struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); + // re-assign field_ref (it was earlier assigned into a field of a generic struct) + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto field_i = v->get_body()->get_field(i); + field_i->mutate()->assign_field_ref(struct_ref->find_field(field_i->get_field_name())); + } + } + + // check that all fields are present (do it after potential generic instantiation, when types are known) + for (StructFieldPtr field_ref : struct_ref->fields) { + if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + bool allow_missing = field_ref->has_default_value() || field_ref->declared_type == TypeDataNever::create(); + if (!allow_missing) { + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + } + } + } + + v->mutate()->assign_struct_ref(struct_ref); + assign_inferred_type(v, TypeDataStruct::create(struct_ref)); + assign_inferred_type(v->get_body(), v); + return ExprFlow(std::move(flow), used_as_condition); + } + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` // more precise is to always set unknown here, but for incorrect usages, instead of an error @@ -1065,12 +1486,12 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - FlowContext process_sequence(V v, FlowContext&& flow) { + FlowContext process_block_statement(V v, FlowContext&& flow) { // we'll print a warning if after some statement, control flow became unreachable // (but don't print a warning if it's already unreachable, for example we're inside always-false if) bool initially_unreachable = flow.is_unreachable(); for (AnyV item : v->get_items()) { - if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->type != ast_empty_statement) { + if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->kind != ast_empty_statement) { v->mutate()->assign_first_unreachable(item); // a warning will be printed later, after type checking } flow = process_any_statement(item, std::move(flow)); @@ -1214,7 +1635,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred - fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); } } @@ -1223,29 +1644,69 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->does_return_self()) { return_unifier.unify_with(fun_ref->parameters[0].declared_type); } - for (AnyExprV return_value : return_statements) { - if (!return_unifier.unify_with(return_value->inferred_type)) { - fire(cur_f, return_value->loc, "can not unify type " + to_string(return_value) + " with previous return type " + to_string(return_unifier.get_result())); - } - } - if (!body_end.is_unreachable()) { - if (!return_unifier.unify_with_implicit_return_void()) { - fire(cur_f, v_function->get_body()->as()->loc_end, "missing return"); - } + bool has_void_returns = false; + bool has_non_void_returns = false; + for (AnyExprV return_expr : return_statements) { + TypePtr cur_type = return_expr->inferred_type; // `return expr` - type of expr; `return` - void + return_unifier.unify_with(cur_type); + has_void_returns |= cur_type == TypeDataVoid::create(); + has_non_void_returns |= cur_type != TypeDataVoid::create(); } inferred_return_type = return_unifier.get_result(); - if (inferred_return_type == nullptr && body_end.is_unreachable()) { + if (inferred_return_type == nullptr) { // if no return statements at all inferred_return_type = TypeDataVoid::create(); } + + if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + if (has_void_returns && has_non_void_returns) { + for (AnyExprV return_expr : return_statements) { + if (return_expr->inferred_type == TypeDataVoid::create()) { + fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function " + to_string(fun_ref)); + } + } + } + if (return_unifier.is_union_of_different_types()) { + // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected + fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); + } } + } else { // asm functions should be strictly typed, this was checked earlier tolk_assert(fun_ref->declared_return_type); } + // visit default values of parameters; to correctly track symbols in `fun f(a: int, b: int = a)`, use flow context + FlowContext params_flow; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + if (param_ref->has_default_value()) { + params_flow = infer_any_expr(param_ref->default_value, std::move(params_flow), false, param_ref->declared_type).out_flow; + } + params_flow.register_known_type(SinkExpression(param_ref), param_ref->declared_type); + } + assign_fun_full_type(fun_ref, inferred_return_type); fun_ref->mutate()->assign_is_type_inferring_done(); } + + // given `const a = 2 + 3` infer that it's int + // for `const a: int = ...` still infer all sub expressions (to be checked in a later pipe) + void start_visiting_constant(GlobalConstPtr const_ref) { + infer_any_expr(const_ref->init_value, FlowContext(), false, const_ref->declared_type); + const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); + } + + // given struct field `a: int = 2 + 3` infer that default value is int, assign inferred_type to all nodes + void start_visiting_struct_fields(StructPtr struct_ref) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + infer_any_expr(field_ref->default_value, FlowContext(), false, field_ref->declared_type); + } + } + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1278,7 +1739,7 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); if (contains) { - fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain; specify `: ` manually"); + fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); } // dig into g's body; it's safe, since the compiler is single-threaded @@ -1289,8 +1750,41 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { called_stack.pop_back(); } +// infer constant type "on demand" +// example: `const a = 1 + b;` +// when analyzing `a`, we need to infer what type const_ref=b has +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { + static std::vector called_stack; + + // prevent recursion like `const a = b; const b = a` + bool contains = std::find(called_stack.begin(), called_stack.end(), const_ref) != called_stack.end(); + if (contains) { + throw ParseError(const_ref->loc, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); + } + + called_stack.push_back(const_ref); + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_constant(const_ref); + called_stack.pop_back(); +} + void pipeline_infer_types_and_calls_and_fields() { visit_ast_of_all_functions(); + + // analyze constants that weren't referenced by any function + InferTypesAndCallsAndFieldsVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + if (!const_ref->inferred_type) { + visitor.start_visiting_constant(const_ref); + } + } + + // infer types for default values in structs + for (StructPtr struct_ref : get_all_declared_structs()) { + if (!struct_ref->is_generic_struct()) { + visitor.start_visiting_struct_fields(struct_ref); + } + } } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp new file mode 100644 index 000000000..64438645b --- /dev/null +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -0,0 +1,1032 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "lazy-helpers.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-visitor.h" +#include "ast-replacer.h" +#include "type-system.h" +#include "smart-casts-cfg.h" +#include "pack-unpack-api.h" + +namespace tolk { + +/* + * This pipe finds `lazy` operators and inserts loading points to load only required fields just before being used. + * It happens after type inferring/checking. While inferring, `lazy expr` was inferred just as expr. + * There is no dedicated `Lazy` type in the type system. All the magic of laziness is calculated here. + * + * This is the second version of the algorithm. The first one (which didn't reach production) attempted to + * calculate precise loading locations right before every field usage. + * E.g., `assert(obj.f1 > 0); assert(obj.f2 > 0)` — it detected "load f1" + assert1 + "load f2" + assert2. + * However, it turned out to increase gas consumption. In practice, when a structure has only a few fields (99% cases), + * it's better to load them ALL AT ONCE rather than on demand: this results in fewer stack manipulations. + * + * `lazy` is not about lazy/partial loading, but also about partial updating. + * As opposed to non-lazy `var st = Storage.fromCell(c); st.xxx = yyy; st.toCell()`, which loads and writes all fields, + * partial updating detects immutable portions of a struct, saves them separately, and reuses on toCell(). + * + * All in all, the algorithm focuses on: + * - Identifying which fields are used for a lazy variable. + * - Loading all required fields at once (and skipping unused ones). + * - Doing lazy matching for unions, to avoid constructing heavy union types on the stack. + * - Calculating and loading used fields at the right place for every union variant. + * - Analyzing modification and gathering immutable portions at loading to be reused on saving. + * + * Example "lazy union": + * > val msg = lazy MyMessage.fromSlice(msgBody); // doesn't construct a union, actually + * > match (msg) { // not a match by type, but a lazy match by a slice prefix + * > CounterReset => { + * > assert(senderAddress == storage.owner) throw 403; + * > // <-- here "load msg.initial" is inserted + * > storage.counter = msg.initial; + * > } + * > } + * + * Example "skip unused": + * > get seqno() { + * > val storage = lazy Storage.fromCell(contract.getData()); + * > // <-- here "skip all fields before seqno, load seqno" is inserted + * > return storage.seqno; + * > } + * + * Example "nesting into try/if": + * > val st = lazy Storage.fromSlice(s); + * > try { + * > // <-- here "load necessary fields" is inserted, if they are used only in `try` body + * > st.someField + * > } + * + * Example "bubbling to closest (lca) statement": + * > val st = lazy Storage.fromSlice(s); + * > // <-- here "load necessary fields" is inserted, because they are used both in `try` and after + * > try { ... st.someField } catch {} + * > st.anotherField + * + * Example "modification and immutable tail": + * > val st = lazy loadStorage(); + * > // <-- here "load f1 f2, save immutable tail, load rest" is inserted + * > ... read all fields + * > st.f2 = newValue; // only f2 is modified, others are only read + * > contract.setData(st.toCell()); // st.toCell() writes f1 f2 and immutable tail + * + * Example "modification and immutable gap": + * > struct Storage { a: int32; b: int32; c: int32; seqno: int32; } + * > var st = lazy loadStorage(); + * > // <-- here "load 96 bits, load seqno" is inserted (note: "abc" are grouped into "96 bits") + * > st.seqno += 1; // only seqno is accessed, "abc" not, that's why they are grouped + * > st.toCell(); // writes 96 bits (grouped "abc") and seqno + * + * Implementation: "original struct" and "hidden struct". + * To group fields for loading/skipping, for every lazy variable, "hidden struct" is created, containing: + * - fields from original struct that are used + * - gaps and artificial fields that are not used, to match binary representation + * + * Example 1: + * | struct Point { x: int8, y: int8 } | hidden: struct lazyPoint { gap: bits8; y: int8 } + * | val p = lazy Point | p is initially `null null` on a stack + * | <-- "skip 8 bits, load y" | field gap skipped, y loaded AND mapped onto a stack to match p.y + * | p.y | p is now `null yValue` + * + * Example 2: + * | struct St { a,b,c; seqno; ... } | hidden: struct lazyStorage { gap: bits96; seqno: int32; tail: slice } + * | val st = lazy St | st is initially `null null null null` + * | <-- "load 96 bits, seqno, save tail" | field gap loaded, seqno loaded, tail (rest fields) saved + * | st.seqno += 1 | st is now `null null null seqno`, "gap" and "tail" kept aside st's ir_idx + * | st.toCell() | writes gap (96 bits, grouped "abc") + modified seqno + storeSlice tail + * + * In a similar way, it works for unions: `lazy UnionType` is represented exactly as `UnionType` on a stack, + * that's why type transitions and methods inlining work natively (when transforming AST to Ops). + * But for each variant, its own "lazyVariant" hidden_struct is created. Used fields are loaded and placed + * into correct placed on a stack, gaps are skipped or placed aside. + * + * Some highlights and considerations: + * - `lazy A.fromSlice(s)` does NOT read from slice immediately; instead, it saves the slice pointer and reads on demand + * (at "loading points" inserted by the compiler in the current pipe). + * - Options can be passed: `lazy A.fromSlice(s, {...})`, but `assertEndAfterReading` is ignored, it doesn't make sense + * (because fields are read later, only required ones, and the last is preloaded rather than loaded). + * There is a special method `a.forceLoadLazyObject()`, can be used inside `match` to load the variant fully. + * - The compiler detects which properties are accessed and inserts "load x y z" as close to the first usage as possible; + * it does NOT split loading into multiple instructions (first x y, then z somewhere below): it has a negative effect. + * - Inlined methods preserve laziness (e.g. `point.getX()` / `storage.save()`): the compiler analyzes the bodies of + * those methods to detect usages of `self`, and marks used fields to be loaded in advance. + * - Perfectly works for unions if a union is used only in `match`; then a union is not even created on a stack: + * instead, this `match` becomes lazy and works but cutting a slice prefix. + * - Effective toCell(): the compiler tracks which fields are modified and reuses an immutable tail slice on writing. + * - Has optimization "to reach a ref, no need to load preceding bits". + * E.g. `struct St { v: int32; content: cell; }` and `st = lazy St; st.content` does only LDREF, no "skip 32 bits". + * - The last requested field is preloaded, not loaded. + * + * There are some drawbacks (probable possible enhancements): + * - When a union is used in some way except match, lazy is inapplicable (compilation error). + * - When a union has some primitives besides structures, lazy is inapplicable (for `T?`, particularly). + * Possible enhancement: handle unions where structs are mixed with primitives. + * - When a struct has a union field, it can be lazily matched only if it's the last. + * Possible enhancement: allow lazy match for union fields in the middle. + * - When `match` is used inside complex expressions, it's not lazy for safety. + * Possible enhancement: `cond && match(...)` is unsafe to be lazy, but `1 + match(...)` could be lazy. + * - Only methods preserve laziness (`p.getX()`), functions do not. + * Possible enhancement: `getXOfPoint(p)` could also be lazy for inlined functions (now `p` is read as a whole). + */ + +// Given fun_ref = "A.fromSlice" from `lazy A.fromSlice(s)` check it's correct to be inside the `lazy` operator. +static bool does_function_satisfy_for_lazy_operator(FunctionPtr fun_ref, bool allow_wrapper = true) { + if (!fun_ref) { + return false; + } + // allow `lazy SomeStruct.fromSlice(s)`; these functions are also handled while transforming AST to Ops + if (fun_ref->is_builtin_function() && fun_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = fun_ref->base_fun_ref->name; + return f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "Cell.load"; + } + // allow `lazy loadData()`, where loadData() is a simple wrapper like + // `fun loadData() { return SomeStruct.fromCell(contract.getData()) }` + if (allow_wrapper && fun_ref->is_code_function() && fun_ref->get_num_params() == 0) { + auto f_body = fun_ref->ast_root->as()->get_body()->as(); + if (f_body->size() == 1) { + if (auto f_returns = f_body->get_item(0)->try_as(); f_returns && f_returns->has_return_value()) { + if (auto f_returns_call = f_returns->get_return_value()->try_as()) { + return does_function_satisfy_for_lazy_operator(f_returns_call->fun_maybe, false); + } + } + } + } + return false; +} + +// Currently, only `A | B | ...` (only structures) can be lazily loaded; later steps rely on StructPtr. +// For example, `(int32, int32) | ...` or `T?` are incompatible with `lazy`. +// If structs don't have prefixes, a prefix tree is built for a union, it also works. +static TypePtr is_union_type_prevented_from_lazy_loading(const TypeDataUnion* t_union) { + for (TypePtr variant : t_union->variants) { + bool is_struct = variant->unwrap_alias()->try_as(); + if (!is_struct) { + return variant; + } + } + return nullptr; +} + +// Given `lazy `, check that expr is correct: a valid function call with valid types. +// If not, fire an error. +static void check_lazy_operator_used_correctly(FunctionPtr cur_f, V v) { + bool is_ok_call = v->get_expr()->kind == ast_function_call + && does_function_satisfy_for_lazy_operator(v->get_expr()->as()->fun_maybe); + if (!is_ok_call) { + fire(cur_f, v->loc, "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); + } + + // it should be either a struct or a union of structs + TypePtr expr_type = v->inferred_type; + if (expr_type->unwrap_alias()->try_as()) { + return; + } + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + if (TypePtr wrong_variant = is_union_type_prevented_from_lazy_loading(expr_union)) { + fire(cur_f, v->loc, "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); + } + return; + } + fire(cur_f, v->loc, "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); +} + +// Given `storage.save()` for a lazy `storage` variable, check if `self` inside should gain laziness. +// If yes, the body of the method is also traversed to detect usages. +// If no, it's assumed that all fields of `storage` are used (an object used "as a whole"). +static bool can_method_be_inlined_preserving_lazy(FunctionPtr method_ref) { + if (method_ref->is_builtin_function() && method_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = method_ref->base_fun_ref->name; + return f_name == "T.toCell" || f_name == "T.forceLoadLazyObject"; + } + + return method_ref->is_inlined_in_place() && // only AST-inlined methods can be lazy + !method_ref->has_mutate_params() && + !method_ref->does_return_self(); +} + +// The first stage of an algorithm is to collect every lazy expression, every field, every union variant. +// This "collecting" is done inside a block, considering all nested statements. +// As a result, we know how many times a variable (and every field independently) is used for reading, writing, etc. +struct ExprUsagesWhileCollecting { + std::string name_str; // "v" / "v.field" / "v.field.nested"; for debugging only + TypePtr expr_type; // either type of variable/field or its narrowed type inside `match` + StructPtr struct_ref; // if it's a struct, otherwise, nullptr + + int used_for_reading = 0; + int used_for_writing = 0; + int used_for_matching = 0; + int used_for_toCell = 0; + int total_usages_with_fields = 0; + std::vector needed_above_stmt; + std::vector> used_as_match_subj; + + std::vector fields; // for struct: every field; otherwise: empty + std::vector variants; // for union: every variant; otherwise: itself (for match over non-union) + + ExprUsagesWhileCollecting(std::string name_for_debugging, TypePtr expr_type, bool is_variant_of_itself = false) + : name_str(std::move(name_for_debugging)) + , expr_type(expr_type) + , struct_ref(nullptr) { + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + variants.reserve(expr_union->size()); + for (int i = 0; i < expr_union->size(); ++i) { + variants.emplace_back(name_str + "(#" + std::to_string(i) + ")", expr_union->variants[i]); + } + return; + } + if (const TypeDataStruct* t_struct = expr_type->unwrap_alias()->try_as()) { + struct_ref = t_struct->struct_ref; + fields.reserve(struct_ref->get_num_fields()); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + fields.emplace_back(name_str + "." + field_ref->name, field_ref->declared_type); + } + } + // to allow code like + // > val msg = lazy Counter.fromSlice(s) <-- struct! not union + // > match (msg) { Counter => {} else => {} } + // we track `msg` inside `match` as a single variant — not over union, but over itself + if (!is_variant_of_itself) { + variants.emplace_back(name_str, expr_type, true); + } + } + + void merge_with_sub_block(const ExprUsagesWhileCollecting& rhs) { + tolk_assert(expr_type->equal_to(rhs.expr_type) && struct_ref == rhs.struct_ref); + used_for_reading += rhs.used_for_reading; + used_for_writing += rhs.used_for_writing; + used_for_matching += rhs.used_for_matching; + used_for_toCell += rhs.used_for_toCell; + total_usages_with_fields += rhs.total_usages_with_fields; + needed_above_stmt.insert(needed_above_stmt.end(), rhs.needed_above_stmt.begin(), rhs.needed_above_stmt.end()); + used_as_match_subj.insert(used_as_match_subj.end(), rhs.used_as_match_subj.begin(), rhs.used_as_match_subj.end()); + for (int i = 0; i < static_cast(fields.size()); ++i) { + fields[i].merge_with_sub_block(rhs.fields[i]); + } + for (int i = 0; i < static_cast(variants.size()); ++i) { + variants[i].merge_with_sub_block(rhs.variants[i]); + } + } + + void on_used_rw(bool is_lvalue) { + is_lvalue ? used_for_writing++ : used_for_reading++; + total_usages_with_fields++; + } + + void on_used_toCell() { + used_for_toCell++; + total_usages_with_fields++; + } + + void on_used_as_match_subj(V v_match) { + used_as_match_subj.push_back(v_match); + used_for_matching++; + total_usages_with_fields++; + } + + bool is_self_or_field_used_for_reading() const { + if (used_for_reading) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + bool is_self_or_field_used_for_toCell() const { + if (used_for_toCell) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_toCell(); + } + return any; + } + + bool is_self_or_child_used_for_writing() const { + if (used_for_writing) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_writing(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_writing(); + } + return any; + } + + bool is_self_or_child_used_for_matching() const { + if (used_for_matching) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_matching(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + void treat_match_like_read() { + if (used_for_matching) { + used_for_reading++; + total_usages_with_fields++; + } + for (ExprUsagesWhileCollecting& field_usages : fields) { + field_usages.treat_match_like_read(); + } + for (ExprUsagesWhileCollecting& variant_usages : variants) { + variant_usages.treat_match_like_read(); + } + } + + LazyStructLoadInfo generate_hidden_struct_load_all(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::vector(struct_ref->fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + std::vector all_fields_load_actions(struct_ref->get_num_fields(), LazyStructLoadInfo::LoadField); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(all_fields_load_actions)); + } + + // for every field of a struct, after calculating all usages, determine: which fields to load, and which to skip + LazyStructLoadInfo calculate_hidden_struct(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + struct FutureField { + LazyStructLoadInfo::ActionWithField action; + std::string_view field_name; + TypePtr field_type; + PackSize pack_size; + + FutureField(LazyStructLoadInfo::ActionWithField action, std::string_view field_name, TypePtr field_type) + : action(action), field_name(field_name), field_type(field_type) + , pack_size(estimate_serialization_size(field_type)) {} + }; + + std::vector future_fields; + + bool object_used_as_a_whole = used_for_reading || used_for_writing || (used_for_toCell && is_variant_of_union); + + // if used as toCell(), detect last modified field_idx: after it, immutable tail can be saved + bool need_immutable_tail = used_for_toCell && !is_variant_of_union; + int last_modified_field_idx = -1; + for (int field_idx = struct_ref->get_num_fields() - 1; field_idx >= 0; --field_idx) { + if (used_for_writing || fields[field_idx].is_self_or_child_used_for_writing()) { + last_modified_field_idx = field_idx; + break; + } + } + + // fill future_fields + for (int field_idx = 0; field_idx < struct_ref->get_num_fields(); ++field_idx) { + StructFieldPtr orig_field = struct_ref->get_field(field_idx); + TypePtr field_type = orig_field->declared_type; + const ExprUsagesWhileCollecting& field_usages = fields[field_idx]; + bool used_anyhow_but_match = field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_writing() || field_usages.is_self_or_field_used_for_toCell(); + + if (need_immutable_tail && field_idx == last_modified_field_idx + 1) { + future_fields.emplace_back(LazyStructLoadInfo::SaveImmutableTail, "(tail)", TypeDataSlice::create()); + } + + if (field_usages.used_for_matching == 1 && !used_anyhow_but_match && !object_used_as_a_whole && !used_for_toCell && !is_variant_of_union && field_idx == struct_ref->get_num_fields() - 1 && + field_type->unwrap_alias()->try_as() && !is_union_type_prevented_from_lazy_loading(field_type->unwrap_alias()->try_as())) { + future_fields.emplace_back(LazyStructLoadInfo::LazyMatchField, orig_field->name, orig_field->declared_type); + continue; + } + if (used_anyhow_but_match || field_usages.is_self_or_child_used_for_matching() || object_used_as_a_whole) { + future_fields.emplace_back(LazyStructLoadInfo::LoadField, orig_field->name, orig_field->declared_type); + continue; + } + + // okay, this field is not needed; we should skip it; + // try to merge "skip 8 bits" + "skip 16 bits" into a single "skip 24 bits" + if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { + if (const TypeDataBitsN* last_bitsN = future_fields.back().field_type->try_as()) { + PackSize cur_size = estimate_serialization_size(field_type); + if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { + TypePtr total_bitsN = TypeDataBitsN::create(last_bitsN->n_width + cur_size.max_bits, true); + future_fields.back().field_type = total_bitsN; + continue; + } + } + } + + // generate "skip 8 bits" instead of "skip int8" (it's more effective, and it can be merged with next) + TypePtr skip_type = field_type; + PackSize skip_size = estimate_serialization_size(field_type); + if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { + skip_type = TypeDataBitsN::create(skip_size.max_bits, true); + } + future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); + } + + // if we need tail, we should load all fields before it (even if they aren't used) + if (need_immutable_tail) { + for (FutureField& f : future_fields) { + if (f.action == LazyStructLoadInfo::SaveImmutableTail) { + break; + } + f.action = LazyStructLoadInfo::LoadField; + } + } + + // here we drop "skip field" if we actually don't need even to skip it, just ignore, like it does not exist; + // example: unused fields in the end `load a; skip b; skip c` -> `load a`; + // example: `skip bits8; load ref` - `load ref`, because to reach a ref, no need to skip preceding bits; + for (size_t i = future_fields.size(); i-- > 0; ) { + if (FutureField f = future_fields[i]; f.action == LazyStructLoadInfo::SkipField) { + PackSize s_cur = f.pack_size; + PackSize s_after(0); + for (size_t j = i + 1; j < future_fields.size(); ++j) { + s_after = EstimateContext::sum(s_after, future_fields[j].pack_size); + } + bool ignore = (s_after.max_bits == 0 && s_after.max_refs == 0) // nothing is loaded after — no need to skip cur + || (s_after.max_bits == 0 && s_cur.max_refs == 0) // no reach ref, no need to skip bits + || (s_after.max_refs == 0 && s_cur.max_bits == 0) // and vice versa: no reach data, to need to skip refs + || (s_cur.max_bits == 0 && s_cur.max_refs == 0); // empty struct/tensor, no need "bits0 skip" + if (ignore) { + future_fields.erase(future_fields.begin() + static_cast(i)); + } + } + } + + // okay, we're done calculating; transform future_fields to hidden_struct + std::vector hidden_fields; + std::vector ith_field_action; + hidden_fields.reserve(future_fields.size()); + ith_field_action.reserve(future_fields.size()); + for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { + FutureField f = future_fields[field_idx]; + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, nullptr, nullptr); + created->mutate()->assign_resolved_type(f.field_type); + hidden_fields.push_back(created); + ith_field_action.push_back(f.action); + } + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::move(hidden_fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(ith_field_action)); + } +}; + +// After collecting all vars/fields/variants usages, we should store, where exactly (in AST) which fields to load. +// Every insertion point is represented as this class, it's transformed to an AST auxiliary vertex by a replacer. +struct OneLoadingInsertionPoint { + std::vector all_stmts_where_used; + TypePtr union_variant; + StructFieldPtr field_ref; + LazyStructLoadInfo load_info; + mutable bool was_inserted_to_ast = false; + + OneLoadingInsertionPoint(std::vector&& all_stmts_where_used, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo&& load_info) + : all_stmts_where_used(std::move(all_stmts_where_used)) + , union_variant(union_variant) + , field_ref(field_ref) + , load_info(std::move(load_info)) {} + + void mark_inserted_to_ast() const { + was_inserted_to_ast = true; + } + + bool is_mentioned_in_stmt(AnyV stmt) const { + return std::find(all_stmts_where_used.begin(), all_stmts_where_used.end(), stmt) != all_stmts_where_used.end(); + } +}; + +// Every `lazy` operator must be assigned to a variable: `var st = lazy getStorage()`. +// Then `st` is a lazy variable, for which all calculations are done, after which it's stored as this class. +struct LazyVarInFunction { + LocalVarPtr var_ref; + V created_by_lazy_op; + V v_lazy_match_var_itself = nullptr; // lazy `match` for the variable itself + V v_lazy_match_last_field = nullptr; // lazy `match` for the last field of a struct + std::vector load_points; // a set of points where AST should be updated + + // convert already calculated usages of "st" variable and all its fields to a final immutable representation + LazyVarInFunction(FunctionPtr cur_f, LocalVarPtr var_ref, V created_by_lazy_op, ExprUsagesWhileCollecting&& var_usages) + : var_ref(var_ref) + , created_by_lazy_op(created_by_lazy_op) { + + // handle if `msg` is used only in `match (msg) { ... }` + // (it may even be not a union, just a struct with opcode, and `match` with `else`) + bool used_only_as_match = var_usages.used_for_matching == 1 && !var_usages.used_for_reading && !var_usages.used_for_toCell && !var_usages.used_for_writing; + if (used_only_as_match) { + v_lazy_match_var_itself = var_usages.used_as_match_subj.front(); + load_points.reserve(var_usages.variants.size()); + for (ExprUsagesWhileCollecting& variant_usages : var_usages.variants) { + LazyStructLoadInfo load_info = variant_usages.calculate_hidden_struct(created_by_lazy_op->loc, true); + load_points.emplace_back(std::move(variant_usages.needed_above_stmt), variant_usages.expr_type, nullptr, std::move(load_info)); + } + return; + } + + // okay, variable is used not only as `match`; + // prohibit this to a union: lazy union may only be matched, nothing more (`msg is A` etc. don't work) + const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); + bool is_union = t_struct == nullptr; + if (is_union) { + fire(cur_f, created_by_lazy_op->loc, "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); + } + + // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used + LazyStructLoadInfo load_info = var_usages.calculate_hidden_struct(created_by_lazy_op->loc, false); + bool is_lazy_match_last_field = !load_info.ith_field_action.empty() && load_info.ith_field_action.back() == LazyStructLoadInfo::LazyMatchField; + load_points.emplace_back(std::move(var_usages.needed_above_stmt), nullptr, nullptr, std::move(load_info)); + + // but probably, there is `match (lazyObj.lastField)` which is lazy; + if (is_lazy_match_last_field) { + StructFieldPtr field_ref = t_struct->struct_ref->fields.back(); + const ExprUsagesWhileCollecting& last_field_usages = var_usages.fields.back(); + v_lazy_match_last_field = last_field_usages.used_as_match_subj.front(); + // inside `match` over a field, loading locations were not detected: insert "load all fields" into every arm + for (int i = 0; i < v_lazy_match_last_field->get_arms_count(); ++i) { + if (auto v_arm = v_lazy_match_last_field->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr union_variant = v_arm->pattern_type_node->resolved_type; + auto v_arm_body = v_arm->get_body()->get_block_statement(); + if (!v_arm_body->empty()) { + const TypeDataUnion* t_union = field_ref->declared_type->unwrap_alias()->try_as(); + int variant_idx = t_union->get_variant_idx(union_variant); + LazyStructLoadInfo load_all = last_field_usages.variants[variant_idx].generate_hidden_struct_load_all(created_by_lazy_op->loc, true); + load_points.emplace_back(std::vector{v_arm_body->get_item(0)}, union_variant, field_ref, std::move(load_all)); + } + } + } + } + } +}; + +static std::unordered_map> functions_with_lazy_vars; + + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block); + +// This visitor finds usages of "v" / "v.field" / etc. in ONE statement or expression and populates lazy_expr data. +// For every struct, all its fields are also populated; for a union — all its variants. +// Since AST vertices don't have "parent_node", we need to remember some details while traversing top-down. +class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { + AnyV cur_stmt; + SinkExpression s_expr; + ExprUsagesWhileCollecting* lazy_expr; + V parent_dot = nullptr; + + CollectUsagesInStatementVisitor(AnyV cur_stmt, SinkExpression s_expr, ExprUsagesWhileCollecting* lazy_expr) + : cur_stmt(cur_stmt), s_expr(s_expr), lazy_expr(lazy_expr) {} + +protected: + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + } + + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + auto backup = parent_dot; + parent_dot = v; + parent::visit(v); + parent_dot = backup; + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (fun_ref && fun_ref->does_accept_self() && can_method_be_inlined_preserving_lazy(fun_ref)) { + AnyExprV dot_obj = v->get_callee()->as()->get_obj(); + if (extract_sink_expression_from_vertex(dot_obj) == s_expr) { + // handle built-in functions specially + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.toCell") { + lazy_expr->on_used_toCell(); + return; + } + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { + lazy_expr->on_used_rw(false); // just use the object as a whole, slice will point to its end + return; + } + if (s_expr.index_path) { // obj.f.method(), mark obj.f as used anyway + lazy_expr->on_used_rw(false); + } + tolk_assert(fun_ref->is_code_function()); + + // okay, we have `st.save()` / `p.getX()`, which will be inlined when transforming to IR; + // dig into that method's body to fetch used fields `self.x` etc. + auto v_body_block = fun_ref->ast_root->try_as()->get_body()->try_as(); + ExprUsagesWhileCollecting inner_usages = collect_expr_usages_in_block(lazy_expr->name_str + "(=self)", SinkExpression(&fun_ref->parameters[0]), lazy_expr->expr_type, v_body_block); + inner_usages.treat_match_like_read(); // nested lazy match in inlined functions doesn't work, it's not wrapped into aux vertex + lazy_expr->merge_with_sub_block(inner_usages); + return; + } + } + + parent::visit(v); + } + + void visit(V v) override { + AnyExprV subj = v->get_subject(); + bool is_match_by_cur = extract_sink_expression_from_vertex(subj) == s_expr; + + // `match` statement over current expression is okay (it will be lazy if it's the only, and other conditions satisfy); + // `match` expression, generally, is not safe to be lazy, e.g. `return cond && match(...)`, + // but simply `return match(...)` / `var result = match(...)` is okay + bool is_safe = false; + if (v->is_statement()) { + is_safe = cur_stmt == v; + } else if (auto v_return = cur_stmt->try_as()) { + is_safe = v_return->get_return_value() == v; + } else if (auto v_assign = cur_stmt->try_as()) { + is_safe = v_assign->get_rhs() == v; + } else if (auto v_set_assign = cur_stmt->try_as()) { + is_safe = v_set_assign->get_rhs() == v; + } + + if (is_match_by_cur && is_safe) { + lazy_expr->on_used_as_match_subj(v); + const TypeDataUnion* expr_as_union = lazy_expr->expr_type->unwrap_alias()->try_as(); + for (int i = 0; i < v->get_arms_count(); ++i) { + if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + auto v_block = v_arm->get_body()->get_block_statement(); + ExprUsagesWhileCollecting variant_usages = collect_expr_usages_in_block(lazy_expr->name_str, s_expr, exact_type, v_block); + int variant_idx = expr_as_union ? expr_as_union->get_variant_idx(exact_type) : 0; // match over non-union is ok + lazy_expr->variants[variant_idx].merge_with_sub_block(variant_usages); + } + } + return; + } + + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static void collect_usages_in_expression(ExprUsagesWhileCollecting* out, SinkExpression s_expr, AnyV v_expr) { + CollectUsagesInStatementVisitor visitor(v_expr, s_expr, out); + visitor.parent::visit(v_expr); + if (out->struct_ref) { + for (int field_idx = 0; field_idx < out->struct_ref->get_num_fields(); ++field_idx) { + collect_usages_in_expression(&out->fields[field_idx], s_expr.get_child_s_expr(field_idx), v_expr); + out->total_usages_with_fields += out->fields[field_idx].total_usages_with_fields; + } + } + } +}; + +// This visitor analyzes A WHOLE BLOCK, statement by statement, and detects all statements where lazy_expr is used. +// It takes care of nested try/catch, etc. +// Ideally, it should calculate the only "lca" AST vertex of all usages, but it's not as easy as it seems. +// Instead, `lazy_expr->needed_above_stmt` contains all statements where expr is "mentioned" (and needs to be loaded before). +// And later, traversing top-down, the first occurrence is taken, inserting an AST aux vertex right before it. +class CollectUsagesInBlockBottomUp { + ExprUsagesWhileCollecting* lazy_expr; + SinkExpression s_expr; + + CollectUsagesInBlockBottomUp(ExprUsagesWhileCollecting* lazy_expr, SinkExpression s_expr) + : lazy_expr(lazy_expr), s_expr(s_expr) {} + + void visit_try_catch_statement(V v) const { + ExprUsagesWhileCollecting u_try = visit_sub_block(v->get_try_body()); + ExprUsagesWhileCollecting u_catch = visit_sub_block(v->get_catch_body()); + + if (u_try.total_usages_with_fields && u_catch.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_try.total_usages_with_fields || u_catch.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_try); + lazy_expr->merge_with_sub_block(u_catch); + } + + void visit_if_statement(V v) const { + ExprUsagesWhileCollecting u_cond = visit_other(v->get_cond()); + ExprUsagesWhileCollecting u_then = visit_sub_block(v->get_if_body()); + ExprUsagesWhileCollecting u_else = visit_sub_block(v->get_else_body()); + + if (u_cond.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields && u_else.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields || u_else.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_cond); + lazy_expr->merge_with_sub_block(u_then); + lazy_expr->merge_with_sub_block(u_else); + } + + void visit_block_statement(V v) const { + for (int i = v->size() - 1; i >= 0; --i) { + AnyV ith_statement = v->get_item(i); + switch (ith_statement->kind) { + case ast_try_catch_statement: + visit_try_catch_statement(ith_statement->as()); + continue; + case ast_if_statement: + visit_if_statement(ith_statement->as()); + continue; + default: + break; + } + + ExprUsagesWhileCollecting u_ith = visit_other(ith_statement); + if (u_ith.total_usages_with_fields) { + u_ith.needed_above_stmt.push_back(ith_statement); + } + lazy_expr->merge_with_sub_block(u_ith); + } + } + + ExprUsagesWhileCollecting visit_sub_block(V v_block) const { + return visit_block_bottom_up(lazy_expr->name_str, s_expr, lazy_expr->expr_type, v_block); + } + + ExprUsagesWhileCollecting visit_other(AnyV v) const { + ExprUsagesWhileCollecting result(lazy_expr->name_str, lazy_expr->expr_type); + CollectUsagesInStatementVisitor::collect_usages_in_expression(&result, s_expr, v); + return result; + } + +public: + static ExprUsagesWhileCollecting visit_block_bottom_up(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + ExprUsagesWhileCollecting lazy_expr(std::move(name_for_debugging), expr_type); + CollectUsagesInBlockBottomUp visitor(&lazy_expr, s_expr); + visitor.visit_block_statement(v_block); + return lazy_expr; + } +}; + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + return CollectUsagesInBlockBottomUp::visit_block_bottom_up(std::move(name_for_debugging), s_expr, expr_type, v_block); +} + +// Step 1: +// This visitor finds `var st = lazy expr`, launches finding usages for `st`, +// and adds `st` as LazyVarInFunction to a global list. +class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + V parent_block = nullptr; + +protected: + void visit(V v) override { + auto backup = parent_block; + parent_block = v; + parent::visit(v); + parent_block = backup; + } + + // `var st = lazy ...` + void visit(V v) override { + if (auto rhs_lazy = v->get_rhs()->try_as()) { + check_lazy_operator_used_correctly(cur_f, rhs_lazy); + + if (auto lhs_var_decl = v->get_lhs()->try_as()) { + auto lhs_var = lhs_var_decl->get_expr()->try_as(); + if (!lhs_var->marked_as_redef) { + // collect usages of a lazy var inside the same block statement where it's declared + LocalVarPtr var_ref = lhs_var->var_ref; + ExprUsagesWhileCollecting var_usages = collect_expr_usages_in_block(var_ref->name, SinkExpression(var_ref), var_ref->declared_type, parent_block); + LazyVarInFunction lazy_var(cur_f, var_ref, rhs_lazy, std::move(var_usages)); + functions_with_lazy_vars[cur_f].emplace_back(std::move(lazy_var)); + } + } + } + + parent::visit(v); + } + + // check that `lazy` operator used in a correct pattern with a correct expression + void visit(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + parent::visit(v); + return; + } + } + + // for `return lazy ...` and other cases except allowed + fire(cur_f, v->loc, "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +// Step 2: +// After visiting all functions and finding all lazy variables, this replacer updates AST, +// inserting (already calculated) load vertices. They are auxiliary vertices holding special data. +// They are handled later when transforming AST to Ops. +class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + V f_body = nullptr; + +protected: + // `var st = lazy expr` -> save "st" (it will be used in codegen to assert "st.x" that "x" is loaded) + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + v->mutate()->assign_dest_var_ref(lazy_var.var_ref); + return parent::replace(v); + } + } + + tolk_assert(false); // all `lazy` operators where detected and handled + } + + // `{ ... }` -> `{ ... load ... }` + AnyV replace(V v) override { + std::vector new_children; // since we don't have "parent_node" and "next_child" in AST, + new_children.reserve(v->size()); // traverse every block statement and insert "load" in the middle + + for (AnyV stmt : v->get_items()) { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + for (const OneLoadingInsertionPoint& ins : lazy_var.load_points) { + if (!ins.was_inserted_to_ast && ins.is_mentioned_in_stmt(stmt)) { + ASTAuxData* aux_data = new AuxData_LazyObjectLoadFields(lazy_var.var_ref, ins.union_variant, ins.field_ref, ins.load_info); + new_children.push_back(createV(stmt->loc, createV(stmt->loc), aux_data, TypeDataVoid::create())); + ins.mark_inserted_to_ast(); + } + } + } + new_children.push_back(parent::replace(stmt)); + } + + v->mutate()->assign_new_children(std::move(new_children)); + return v; + } + + // `match (lazy_obj)` / `match (lazy_obj.field)` -> wrap with aux + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + bool is_lazy_match_for_union = lazy_var.v_lazy_match_var_itself == v; + if (is_lazy_match_for_union) { + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, nullptr); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + + bool is_lazy_match_for_last_field = lazy_var.v_lazy_match_last_field == v; + if (is_lazy_match_for_last_field) { + StructPtr struct_ref = lazy_var.var_ref->declared_type->unwrap_alias()->try_as()->struct_ref; + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, struct_ref->fields.back()); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + f_body = v_function->get_body()->as(); + parent::replace(v_function->get_body()); + } +}; + +// Step 3: +// After modifying AST (inserting loads, lazy match, etc.), +// check __expect_lazy() calls, used in compiler tests as assertions. +class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static std::string stringify_lazy_load_above_stmt(const AuxData_LazyObjectLoadFields* aux_load) { + static const char* action_to_str[] = {"load", "skip", "lazy match", "save immutable"}; + + const LazyStructLoadInfo& load_info = aux_load->load_info; + StructPtr struct_ref = load_info.hidden_struct; + std::string_view last_action; + + std::string result = "[" + aux_load->var_ref->name + "] "; + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + std::string field_name = struct_ref->get_field(i)->name; + if (field_name == "(gap)") { + field_name = "(" + struct_ref->get_field(i)->declared_type->as_human_readable() + ")"; + } + std::string_view action = action_to_str[load_info.ith_field_action[i]]; + if (action != last_action) { + if (result[result.size() - 2] != ']') { + result += ", "; + } + result += action; + last_action = action; + } + result += " "; + result += field_name; + } + return result; + } + + void visit(V v) override { + // again, given "__expect_lazy(...)", we have no "next sibling", so traverse block statements + for (int i = 0; i < v->size(); ++i) { + AnyV cur_stmt = v->get_item(i); + if (auto v_call = cur_stmt->try_as()) { + if (v_call->fun_maybe && v_call->fun_maybe->is_builtin_function() && v_call->fun_maybe->name == "__expect_lazy") { + AnyV next_stmt = v->get_item(i + 1); + std::string_view expected = v_call->get_arg(0)->get_expr()->as()->str_val; + std::string actual; + if (auto next_aux = next_stmt->try_as()) { + if (const auto* aux_load = dynamic_cast(next_aux->aux_data)) { + actual = stringify_lazy_load_above_stmt(aux_load); + } + if (const auto* aux_match = dynamic_cast(next_aux->aux_data)) { + actual = "[" + aux_match->var_ref->name + "] " + "lazy match"; + } + } + + if (actual != expected) { + throw ParseError(cur_stmt->loc, "__expect_lazy failed: actual \"" + actual + "\""); + } + } + } + parent::visit(cur_stmt); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_lazy_load_insertions() { + visit_ast_of_all_functions(); + replace_ast_of_all_functions(); + visit_ast_of_all_functions(); + functions_with_lazy_vars.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index c4c5d1dcf..6b053915f 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -53,10 +53,33 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { auto v_not = createV(loc, "!", tok_logical_not, rhs); v_not->assign_inferred_type(TypeDataBool::create()); v_not->assign_rvalue_true(); - v_not->assign_fun_ref(lookup_global_symbol("!b_")->try_as()); + v_not->assign_fun_ref(lookup_function("!b_")); return v_not; } + static bool expect_integer(TypePtr inferred_type) { + if (inferred_type == TypeDataInt::create()) { + return true; + } + if (inferred_type->try_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; + } + + static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; + } + protected: AnyExprV replace(V v) override { @@ -66,16 +89,16 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (auto inner_not = v->get_rhs()->try_as(); inner_not && inner_not->tok == tok_logical_not) { AnyExprV cond_not_not = inner_not->get_rhs(); // `!!boolVar` => `boolVar` - if (cond_not_not->inferred_type == TypeDataBool::create()) { + if (expect_boolean(cond_not_not->inferred_type)) { return cond_not_not; } // `!!intVar` => `intVar != 0` - if (cond_not_not->inferred_type == TypeDataInt::create()) { + if (expect_integer(cond_not_not->inferred_type)) { auto v_zero = create_int_const(v->loc, td::make_refint(0)); auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); - v_neq->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_neq->mutate()->assign_fun_ref(lookup_function("_!=_")); return v_neq; } } @@ -94,7 +117,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v->tok == tok_eq || v->tok == tok_neq) { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); - if (lhs->inferred_type == TypeDataBool::create() && rhs->type == ast_bool_const) { + if (expect_boolean(lhs->inferred_type) && rhs->kind == ast_bool_const) { // `boolVar == true` / `boolVar != false` if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { return lhs; @@ -118,9 +141,18 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); } // `if (x != null)` -> ifnot(x == null) - if (auto v_cond_isnull = v->get_cond()->try_as(); v_cond_isnull && v_cond_isnull->is_negated) { - v_cond_isnull->mutate()->assign_is_negated(!v_cond_isnull->is_negated); - v = createV(v->loc, !v->is_ifnot, v_cond_isnull, v->get_if_body(), v->get_else_body()); + if (auto v_cond_istype = v->get_cond()->try_as(); v_cond_istype && v_cond_istype->is_negated) { + v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); + v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); + } + // `if (addr1 != addr2)` -> ifnot(addr1 == addr2) + if (auto v_cond_neq = v->get_cond()->try_as()) { + if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + auto v_cond_eq = createV(v_cond_neq->loc, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); + v_cond_eq->mutate()->assign_inferred_type(v_cond_neq->inferred_type); + v_cond_eq->mutate()->assign_rvalue_true(); + v = createV(v->loc, !v->is_ifnot, v_cond_eq, v->get_if_body(), v->get_else_body()); + } } return v; diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index a8b4f1ae5..3e02cf773 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -34,30 +34,22 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { - std::string arg_str(arg_expr->type == ast_reference ? arg_expr->as()->get_name() : "obj"); +static void fire_error_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { + std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); - // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` - if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { - v->error("`" + fun_ref->name + "` is a mutating method; consider calling `" + arg_str + "." + fun_ref->name + "()`, not `" + fun_ref->name + "(" + arg_str + ")`"); - } - // case: `cs.mutating_function()`; suggest: `mutating_function(mutate cs)` or make it a method - if (p_sym.is_mutate_parameter() && called_as_method && p_sym.param_idx == 0 && !fun_ref->does_accept_self()) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; consider calling `" + fun_ref->name + "(mutate " + arg_str + ")`, not `" + arg_str + "." + fun_ref->name + "`(); alternatively, rename parameter to `self` to make it a method"); - } - // case: `mutating_function(arg)`; suggest: `mutate arg` if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; you need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + // called `mutating_function(arg)`; suggest: `mutate arg` + fire(cur_f, loc, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + } else { + // called `usual_function(mutate arg)` + fire(cur_f, loc, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); } - // case: `usual_function(mutate arg)` - if (!p_sym.is_mutate_parameter() && arg_passed_as_mutate) { - v->error("incorrect `mutate`, since `" + fun_ref->name + "` does not mutate this parameter"); - } - throw Fatal("unreachable"); } class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + void visit(V v) override { // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` FunctionPtr fun_ref = v->fun_maybe; @@ -66,41 +58,35 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod for (int i = 0; i < v->get_num_args(); ++i) { auto v_arg = v->get_arg(i); if (v_arg->passed_as_mutate) { - v_arg->error("`mutate` used for non-mutate argument"); + v_arg->error("`mutate` used for non-mutate parameter"); } } return; } - int delta_self = v->is_dot_call(); - tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); - - if (v->is_dot_call()) { - if (fun_ref->does_mutate_self()) { - // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self - // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue - // (it will be extracted as tmp var when transforming AST to IR) - AnyExprV leftmost_obj = v->get_dot_obj(); - while (true) { - if (auto as_par = leftmost_obj->try_as()) { - leftmost_obj = as_par->get_expr(); - } else if (auto as_cast = leftmost_obj->try_as()) { - leftmost_obj = as_cast->get_expr(); - } else if (auto as_nn = leftmost_obj->try_as()) { - leftmost_obj = as_nn->get_expr(); - } else { - break; - } - } - bool will_be_extracted_as_tmp_var = leftmost_obj->type == ast_function_call; - if (!will_be_extracted_as_tmp_var) { - leftmost_obj->mutate()->assign_lvalue_true(); - v->get_dot_obj()->mutate()->assign_lvalue_true(); + int delta_self = v->get_self_obj() != nullptr; + tolk_assert(fun_ref->get_num_params() >= delta_self + v->get_num_args()); + + if (delta_self && fun_ref->does_mutate_self()) { + // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self + // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue + // (it will be extracted as tmp var when transforming AST to IR) + AnyExprV leftmost_obj = v->get_self_obj(); + while (true) { + if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; } } - - if (!fun_ref->does_accept_self() && fun_ref->parameters[0].is_mutate_parameter()) { - fire_error_invalid_mutate_arg_passed(v, fun_ref, fun_ref->parameters[0], true, false, v->get_dot_obj()); + bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; + if (!will_be_extracted_as_tmp_var) { + leftmost_obj->mutate()->assign_lvalue_true(); + v->get_self_obj()->mutate()->assign_lvalue_true(); } } @@ -108,7 +94,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; auto arg_i = v->get_arg(i); if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { - fire_error_invalid_mutate_arg_passed(arg_i, fun_ref, p_sym, false, arg_i->passed_as_mutate, arg_i->get_expr()); + fire_error_invalid_mutate_arg_passed(cur_f, arg_i->loc, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); } parent::visit(arg_i); } @@ -119,6 +105,11 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_refine_lvalue_for_mutate_arguments() { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 45246d6b9..917722946 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -19,10 +19,9 @@ #include "src-file.h" #include "ast.h" #include "compiler-state.h" -#include "constant-evaluator.h" #include "generics-helpers.h" +#include "pack-unpack-serializers.h" #include "td/utils/crypto.h" -#include "type-system.h" #include /* @@ -37,7 +36,7 @@ namespace tolk { -static int calculate_method_id_for_entrypoint(std::string_view func_name) { +static int calculate_tvm_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "main" || func_name == "onInternalMessage") { return 0; } @@ -53,18 +52,18 @@ static int calculate_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "onSplitInstall") { return -4; } + if (func_name == "onBouncedMessage") { + return FunctionData::EMPTY_TVM_METHOD_ID; + } tolk_assert(false); } -static int calculate_method_id_by_func_name(std::string_view func_name) { +static int calculate_tvm_method_id_by_func_name(std::string_view func_name) { unsigned int crc = td::crc16(static_cast(func_name)); return static_cast(crc & 0xffff) | 0x10000; } -static void validate_arg_ret_order_of_asm_function(V v_body, int n_params, TypePtr ret_type) { - if (!ret_type) { - v_body->error("asm function must declare return type (before asm instructions)"); - } +static void validate_arg_ret_order_of_asm_function(V v_body, int n_params) { if (n_params > 16) { v_body->error("asm function can have at most 16 parameters"); } @@ -96,52 +95,90 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } } -static const GenericsDeclaration* construct_genericTs(V v_list) { - std::vector itemsT; - itemsT.reserve(v_list->size()); - - for (int i = 0; i < v_list->size(); ++i) { - auto v_item = v_list->get_item(i); - auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::GenericsItem& prev) { - return prev.nameT == v_item->nameT; - }); - if (it_existing != itemsT.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); - } - itemsT.emplace_back(v_item->nameT); - } - - return new GenericsDeclaration(std::move(itemsT)); -} - -static void register_constant(V v) { - ConstantValue init_value = eval_const_init_value(v->get_init_value()); - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, std::move(init_value)); - - if (v->declared_type) { - bool ok = (c_sym->is_int_const() && (v->declared_type == TypeDataInt::create())) - || (c_sym->is_slice_const() && (v->declared_type == TypeDataSlice::create())); - if (!ok) { - v->error("expression type does not match declared type"); - } - } +static GlobalConstPtr register_constant(V v) { + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); v->mutate()->assign_const_ref(c_sym); + return c_sym; } -static void register_global_var(V v) { - GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->declared_type); +static GlobalVarPtr register_global_var(V v) { + GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); G.symtable.add_global_var(g_sym); G.all_global_vars.push_back(g_sym); - v->mutate()->assign_var_ref(g_sym); + v->mutate()->assign_glob_ref(g_sym); + return g_sym; +} + +static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving + AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` + + G.symtable.add_type_alias(a_sym); + v->mutate()->assign_alias_ref(a_sym); + return a_sym; +} + +static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + auto v_body = v->get_struct_body(); + + std::vector fields; + fields.reserve(v_body->get_num_fields()); + for (int i = 0; i < v_body->get_num_fields(); ++i) { + auto v_field = v_body->get_field(i); + std::string field_name = static_cast(v_field->get_identifier()->name); + + for (StructFieldPtr prev : fields) { + if (UNLIKELY(prev->name == field_name)) { + v_field->error("redeclaration of field `" + field_name + "`"); + } + } + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->type_node, v_field->default_value)); + } + + PackOpcode opcode(0, 0); + if (v->has_opcode()) { + auto v_opcode = v->get_opcode()->as(); + if (v_opcode->intval < 0 || v_opcode->intval > (1ULL << 48)) { + v->error("opcode must not exceed 2^48"); + } + opcode.pack_prefix = v_opcode->intval->to_long(); + + std::string_view prefix_str = v_opcode->orig_str; + if (prefix_str.starts_with("0x")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2) * 4; + } else if (prefix_str.starts_with("0b")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2); + } else { + tolk_assert(false); + } + } + + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving + StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); + s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` + + G.symtable.add_struct(s_sym); + G.all_structs.push_back(s_sym); + v->mutate()->assign_struct_ref(s_sym); + return s_sym; } static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { - return {"", v->loc, v->declared_type, 0, idx}; + return LocalVarData{"", v->loc, v->type_node, v->default_value, 0, idx}; } int flags = 0; @@ -151,76 +188,76 @@ static LocalVarData register_parameter(V v, int idx) { if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->declared_type, flags, idx); + return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, v->default_value, flags, idx); } -static void register_function(V v) { - std::string_view func_name = v->get_identifier()->name; +static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + if (v->is_builtin_function()) { + return nullptr; + } + + std::string_view f_identifier = v->get_identifier()->name; // function or method name - // calculate TypeData of a function - std::vector arg_types; std::vector parameters; - int n_params = v->get_num_params(); int n_mutate_params = 0; - arg_types.reserve(n_params); - parameters.reserve(n_params); - for (int i = 0; i < n_params; ++i) { + parameters.reserve(v->get_num_params()); + for (int i = 0; i < v->get_num_params(); ++i) { auto v_param = v->get_param(i); - arg_types.emplace_back(v_param->declared_type); parameters.emplace_back(register_parameter(v_param, i)); n_mutate_params += static_cast(v_param->declared_as_mutate); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list) { - genericTs = construct_genericTs(v->genericsT_list); - } - if (v->is_builtin_function()) { - const Symbol* sym = lookup_global_symbol(func_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref || !fun_ref->is_builtin_function()) { - v->error("`builtin` used for non-builtin function"); - } - v->mutate()->assign_fun_ref(fun_ref); - return; + std::string method_name; + if (v->receiver_type_node) { + method_name = f_identifier; } - - if (G.is_verbosity(1) && v->is_code_function()) { - std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; + std::string name = std::move(override_name); + if (name.empty()) { + name = f_identifier; } - FunctionBody f_body = v->get_body()->type == ast_sequence ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving + FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, v->inline_mode, genericTs, substitutedTs, f_body, v); + f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` - if (const auto* v_asm = v->get_body()->try_as()) { - validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params(), v->declared_return_type); + if (auto v_asm = v->get_body()->try_as()) { + if (!v->return_type_node) { + v_asm->error("asm function must declare return type (before asm instructions)"); + } + validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params()); f_sym->arg_order = v_asm->arg_order; f_sym->ret_order = v_asm->ret_order; } - if (v->method_id.not_null()) { - f_sym->method_id = static_cast(v->method_id->to_long()); - } else if (v->flags & FunctionData::flagGetMethod) { - f_sym->method_id = calculate_method_id_by_func_name(func_name); - for (FunctionPtr other : G.all_get_methods) { - if (other->method_id == f_sym->method_id) { + if (v->tvm_method_id.not_null()) { + f_sym->tvm_method_id = static_cast(v->tvm_method_id->to_long()); + } else if (v->flags & FunctionData::flagContractGetter) { + f_sym->tvm_method_id = calculate_tvm_method_id_by_func_name(f_identifier); + for (FunctionPtr other : G.all_contract_getters) { + if (other->tvm_method_id == f_sym->tvm_method_id) { v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); } } } else if (v->flags & FunctionData::flagIsEntrypoint) { - f_sym->method_id = calculate_method_id_for_entrypoint(func_name); + f_sym->tvm_method_id = calculate_tvm_method_id_for_entrypoint(f_identifier); } f_sym->flags |= v->flags; if (n_mutate_params) { f_sym->flags |= FunctionData::flagHasMutateParams; } - G.symtable.add_function(f_sym); + if (!f_sym->receiver_type_node) { + G.symtable.add_function(f_sym); + } else if (!substitutedTs) { + G.all_methods.push_back(f_sym); + } G.all_functions.push_back(f_sym); - if (f_sym->is_get_method()) { - G.all_get_methods.push_back(f_sym); + if (f_sym->is_contract_getter()) { + G.all_contract_getters.push_back(f_sym); } v->mutate()->assign_fun_ref(f_sym); + return f_sym; } static void iterate_through_file_symbols(const SrcFile* file) { @@ -231,7 +268,7 @@ static void iterate_through_file_symbols(const SrcFile* file) { tolk_assert(file && file->ast); for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - switch (v->type) { + switch (v->kind) { case ast_import_directive: // on `import "another-file.tolk"`, register symbols from that file at first // (for instance, it can calculate constants, which are used in init_val of constants in current file below import) @@ -244,6 +281,12 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_global_var_declaration: register_global_var(v->as()); break; + case ast_type_alias_declaration: + register_type_alias(v->as()); + break; + case ast_struct_declaration: + register_struct(v->as()); + break; case ast_function_declaration: register_function(v->as()); break; @@ -259,4 +302,19 @@ void pipeline_register_global_symbols() { } } +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_function(v, base_fun_ref, std::move(name), substitutedTs); +} + +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_struct(v, base_struct_ref, std::move(name), substitutedTs); +} + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_type_alias(v, base_alias_ref, std::move(name), substitutedTs); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 5a735885e..46710cb63 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -18,32 +18,21 @@ #include "platform-utils.h" #include "compiler-state.h" #include "src-file.h" -#include "generics-helpers.h" #include "ast.h" #include "ast-visitor.h" -#include "type-system.h" #include /* - * This pipe resolves identifiers (local variables and types) in all functions bodies. + * This pipe resolves identifiers (local variables, globals, constants, etc.) in all functions bodies. * It happens before type inferring, but after all global symbols are registered. * It means, that for any symbol `x` we can look up whether it's a global name or not. * - * About resolving variables. * Example: `var x = 10; x = 20;` both `x` point to one LocalVarData. * Example: `x = 20` undefined symbol `x` is also here (unless it's a global) * Variables scoping and redeclaration are also here. * Note, that `x` is stored as `ast_reference (ast_identifier "x")`. More formally, "references" are resolved. * "Reference" in AST, besides the identifier, stores optional generics instantiation. `x` is grammar-valid. * - * About resolving types. At the moment of parsing, `int`, `cell` and other predefined are parsed as TypeDataInt, etc. - * All the others are stored as TypeDataUnresolved, to be resolved here, after global symtable is filled. - * Example: `var x: T = 0` unresolved "T" is replaced by TypeDataGenericT inside `f`. - * Example: `f()` unresolved "MyAlias" is replaced by TypeDataAlias inside the reference. - * Example: `fun f(): KKK` unresolved "KKK" fires an error "unknown type name". - * When structures and type aliases are implemented, their resolving will also be done here. - * See finalize_type_data(). - * * Note, that functions/methods binding is NOT here. * In other words, for ast_function_call `beginCell()` and `t.tupleAt(0)`, their fun_ref is NOT filled here. * Functions/methods binding is done later, simultaneously with type inferring and generics instantiation. @@ -53,7 +42,6 @@ * As a result of this step, * * every V::sym is filled, pointing either to a local var/parameter, or to a global symbol * (exceptional for function calls and methods, their references are bound later) - * * all TypeData in all symbols is ready for analyzing, TypeDataUnresolved won't occur later in pipeline */ namespace tolk { @@ -61,30 +49,18 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { if (v->name == "self") { - throw ParseError(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + fire(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); } else { - throw ParseError(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + fire(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); } } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, const std::string &text) { - throw ParseError(cur_f, loc, "unknown type name `" + text + "`"); -} - -static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { - SrcLocation sym_loc = used_sym->loc; - if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { - const SrcFile* declared_in = sym_loc.get_src_file(); - bool has_import = false; - for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) { - if (import.imported_file == declared_in) { - has_import = true; - } - } - if (!has_import) { - throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?"); - } +static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V v) { + if (v->name == "random") { // calling `random()`, but it's a struct, correct is `random.uint256()` + fire(cur_f, v->loc, "`random` is not a function, you probably want `random.uint256()`"); + } else { + fire(cur_f, v->loc, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); } } @@ -103,7 +79,7 @@ struct NameAndScopeResolver { void close_scope([[maybe_unused]] SrcLocation loc) { // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; if (UNLIKELY(scopes.empty())) { - throw Fatal{"cannot close the outer scope"}; + throw Fatal("cannot close the outer scope"); } scopes.pop_back(); } @@ -135,51 +111,20 @@ struct NameAndScopeResolver { } }; -struct TypeDataResolver { - GNU_ATTRIBUTE_NOINLINE - static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { - if (const TypeDataUnresolved* un = child->try_as()) { - if (genericTs && genericTs->has_nameT(un->text)) { - std::string nameT = un->text; - return TypeDataGenericT::create(std::move(nameT)); - } - if (un->text == "auto") { - throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); - } - if (un->text == "self") { - throw ParseError(cur_f, un->loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); - } - fire_error_unknown_type_name(cur_f, un->loc, un->text); - } - return child; - }); - } -}; - -static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - if (!type_data || !type_data->has_unresolved_inside()) { - return type_data; - } - return TypeDataResolver::resolve_identifiers_in_type_data(cur_f, type_data, genericTs); -} - - class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // more correctly this field shouldn't be static, but currently there is no need to make it a part of state static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static const GenericsDeclaration* current_genericTs; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, TypePtr declared_type, bool immutable) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type, immutable * LocalVarData::flagImmutable, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } static void process_catch_variable(AnyExprV catch_var) { if (auto v_ref = catch_var->try_as()) { - LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true); + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true, false); v_ref->mutate()->assign_sym(var_ref); } } @@ -197,9 +142,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } v->mutate()->assign_var_ref(var_ref); } else { - TypePtr declared_type = finalize_type_data(cur_f, v->declared_type, current_genericTs); - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, declared_type, v->is_immutable); - v->mutate()->assign_resolved_type(declared_type); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable, v->is_lateinit); v->mutate()->assign_var_ref(var_ref); } } @@ -214,42 +157,72 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym) { fire_error_undefined_symbol(cur_f, v->get_identifier()); } + if (sym->try_as() || sym->try_as()) { + fire_error_type_used_as_symbol(cur_f, v->get_identifier()); + } v->mutate()->assign_sym(sym); // for global functions, global vars and constants, `import` must exist if (!sym->try_as()) { - check_import_exists_when_using_sym(cur_f, v, sym); - } - - // for `f` / `f`, resolve "MyAlias" and "T" - // (for function call `f()`, this v (ast_reference `f`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); } } } void visit(V v) override { - // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" - // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + try { + parent::visit(v->get_obj()); + } catch (const ParseError&) { + if (v->get_obj()->kind == ast_reference) { + // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int + // suppress this exception till a later pipe, it will be tried to be resolved as a type + return; } + throw; } - parent::visit(v->get_obj()); } - void visit(V v) override { - TypePtr cast_to_type = finalize_type_data(cur_f, v->cast_to_type, current_genericTs); - v->mutate()->assign_resolved_type(cast_to_type); - parent::visit(v->get_expr()); + void visit(V v) override { + current_scope.open_scope(v->loc); + parent::visit(v->get_block_statement()); + current_scope.close_scope(v->loc); } - void visit(V v) override { + void visit(V v) override { + current_scope.open_scope(v->loc); // `match (var a = init_val) { ... }` + parent::visit(v); // then `a` exists only inside `match` arms + current_scope.close_scope(v->loc); + } + + void visit(V v) override { + // resolve identifiers after => at first + visit(v->get_body()); + // because handling lhs of => is comprehensive + + switch (v->pattern_kind) { + case MatchArmKind::exact_type: { + if (auto maybe_ident = v->pattern_type_node->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && sym->try_as()) { + auto v_ident = createV(v->loc, sym->name); + AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); + parent::visit(pattern_expr); + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, pattern_expr); + } + } + break; + } + case MatchArmKind::const_expression: { + parent::visit(v->get_pattern_expr()); + break; + } + default: + // for `else` match branch, do nothing: its body was already traversed above + break; + } + } + + void visit(V v) override { if (v->empty()) { return; } @@ -278,66 +251,63 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { public: bool should_visit_function(FunctionPtr fun_ref) override { - // this pipe is done just after parsing - // visit both asm and code functions, resolve identifiers in parameter/return types everywhere - // for generic functions, unresolved "T" will be replaced by TypeDataGenericT - return true; + return fun_ref->is_code_function(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { cur_f = fun_ref; - current_genericTs = fun_ref->genericTs; - - for (int i = 0; i < v->get_num_params(); ++i) { - const LocalVarData& param_var = fun_ref->parameters[i]; - TypePtr declared_type = finalize_type_data(cur_f, param_var.declared_type, fun_ref->genericTs); - v->get_param(i)->mutate()->assign_param_ref(¶m_var); - v->get_param(i)->mutate()->assign_resolved_type(declared_type); - param_var.mutate()->assign_resolved_type(declared_type); - } - TypePtr return_type = finalize_type_data(cur_f, fun_ref->declared_return_type, fun_ref->genericTs); - v->mutate()->assign_resolved_type(return_type); - fun_ref->mutate()->assign_resolved_type(return_type); - - if (fun_ref->is_code_function()) { - auto v_seq = v->get_body()->as(); - current_scope.open_scope(v->loc); - for (int i = 0; i < v->get_num_params(); ++i) { - current_scope.add_local_var(&fun_ref->parameters[i]); + + auto v_block = v->get_body()->as(); + current_scope.open_scope(v->loc); + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->parameters[i]; + current_scope.add_local_var(param_ref); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); } - parent::visit(v_seq); - current_scope.close_scope(v_seq->loc_end); - tolk_assert(current_scope.scopes.empty()); } + parent::visit(v_block); + current_scope.close_scope(v_block->loc_end); + tolk_assert(current_scope.scopes.empty()); - current_genericTs = nullptr; cur_f = nullptr; } + + void start_visiting_constant(GlobalConstPtr const_ref) { + // `const a = b`, resolve `b` + parent::visit(const_ref->init_value); + } + + void start_visiting_struct_fields(StructPtr struct_ref) { + // field `a: int = C`, resolve `C` + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; -const GenericsDeclaration* AssignSymInsideFunctionVisitor::current_genericTs = nullptr; void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; for (const SrcFile* file : G.all_src_files) { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - if (auto v_func = v->try_as()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { tolk_assert(v_func->fun_ref); - visitor.start_visiting_function(v_func->fun_ref, v_func); - - } else if (auto v_global = v->try_as()) { - TypePtr declared_type = finalize_type_data(nullptr, v_global->var_ref->declared_type, nullptr); - v_global->mutate()->assign_resolved_type(declared_type); - v_global->var_ref->mutate()->assign_resolved_type(declared_type); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); + } } else if (auto v_const = v->try_as()) { - if (v_const->declared_type) { - TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); - v_const->mutate()->assign_resolved_type(declared_type); - v_const->const_ref->mutate()->assign_resolved_type(declared_type); - } + tolk_assert(v_const->const_ref); + visitor.start_visiting_constant(v_const->const_ref); + + } else if (auto v_struct = v->try_as()) { + tolk_assert(v_struct->struct_ref); + visitor.start_visiting_struct_fields(v_struct->struct_ref); } } } @@ -350,4 +320,8 @@ void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr fun_ref) { } } +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr struct_ref) { + AssignSymInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp new file mode 100644 index 000000000..7ca035cfd --- /dev/null +++ b/tolk/pipe-resolve-types.cpp @@ -0,0 +1,682 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include "ast.h" +#include "ast-visitor.h" +#include "generics-helpers.h" +#include "type-system.h" +#include + +namespace tolk { + +/* + * This pipe transforms AST of types into TypePtr. + * It happens after all global symbols were registered, and all local references were bound. + * + * At the moment of parsing, `int`, `cell` and other were parsed as AnyTypeV (ast_type_leaf_text and others). + * Example: `var x: int = ...` to TypeDataInt + * Example: `fun f(a: cell): (int, User)` param to TypeDataCell, return type to TypeDataTensor(TypeDataInt, TypeDataStruct) + * Example: `var x: T = 0` to TypeDataGenericT inside `f` + * Example: `f()` to TypeDataAlias inside instantiation list + * Example: `arg: Wrapper` instantiates "Wrapper" right here and returns TypeDataStruct to it + * Example: `fun f(): KKK` fires an error "unknown type name" + * + * Types resolving is done everywhere: inside functions bodies, in struct fields, inside globals declaration, etc. + * See finalize_type_node(). + * + * Note, that resolving T to TypeDataGenericT (and replacing T with substitution when instantiating a generic type) + * is also done here, see genericTs and substitutedTs. + * Note, that instantiating generic structs and aliases is also done here (if they don't have generic Ts inside). + * Example: `type OkInt = Ok`, struct "Ok" is instantiated (as a clone of `Ok` substituting T=int) + * Example: `type A = Ok`, then `Ok` is not ready yet, it's left as TypeDataGenericTypeWithTs. + */ + +static std::unordered_map visited_structs; +static std::unordered_map visited_aliases; + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { + if (text == "auto") { + fire(cur_f, loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + } + if (text == "self") { + fire(cur_f, loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + } + fire(cur_f, loc, "unknown type name `" + static_cast(text) + "`"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocation loc, TypePtr disallowed_variant) { + fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocation loc, const std::string& type_name_with_Ts) { + fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); +} + +static TypePtr parse_intN_uintN(std::string_view strN, bool is_unsigned) { + int n; + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 257 - static_cast(is_unsigned)) { + return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataIntN::create(n, is_unsigned, false); +} + +static TypePtr parse_bytesN_bitsN(std::string_view strN, bool is_bits) { + int n; + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 1024) { + return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataBitsN::create(n, is_bits); +} + +static TypePtr try_parse_predefined_type(std::string_view str) { + switch (str.size()) { + case 3: + if (str == "int") return TypeDataInt::create(); + break; + case 4: + if (str == "cell") return TypeDataCell::create(); + if (str == "void") return TypeDataVoid::create(); + if (str == "bool") return TypeDataBool::create(); + if (str == "null") return TypeDataNullLiteral::create(); + break; + case 5: + if (str == "slice") return TypeDataSlice::create(); + if (str == "tuple") return TypeDataTuple::create(); + if (str == "coins") return TypeDataCoins::create(); + if (str == "never") return TypeDataNever::create(); + break; + case 7: + if (str == "builder") return TypeDataBuilder::create(); + if (str == "address") return TypeDataAddress::create(); + break; + case 8: + if (str == "varint16") return TypeDataIntN::create(16, false, true); + if (str == "varint32") return TypeDataIntN::create(32, false, true); + break; + case 9: + if (str == "varuint16") return TypeDataIntN::create(16, true, true); + if (str == "varuint32") return TypeDataIntN::create(32, true, true); + break; + case 12: + if (str == "continuation") return TypeDataContinuation::create(); + break; + default: + break; + } + + if (str.starts_with("int")) { + if (TypePtr intN = parse_intN_uintN(str.substr(3), false)) { + return intN; + } + } + if (str.starts_with("uint")) { + if (TypePtr uintN = parse_intN_uintN(str.substr(4), true)) { + return uintN; + } + } + if (str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN_bitsN(str.substr(4), true)) { + return bitsN; + } + } + if (str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN_bitsN(str.substr(5), false)) { + return bytesN; + } + } + + return nullptr; +} + +class TypeNodesVisitorResolver { + FunctionPtr cur_f; // exists if we're inside its body + const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` + const GenericsSubstitutions* substitutedTs; // `T=int` if we're inside `f` + bool treat_unresolved_as_genericT; // used for receivers `fun Container.create()`, T becomes generic + + TypePtr parse_ast_type_node(AnyTypeV v, bool allow_without_type_arguments) { + switch (v->kind) { + case ast_type_leaf_text: { + std::string_view text = v->as()->text; + SrcLocation loc = v->as()->loc; + if (genericTs && genericTs->find_nameT(text) != -1) { + // if we're inside `f`, replace "T" with TypeDataGenericT + return TypeDataGenericT::create(static_cast(text)); + } + if (substitutedTs && substitutedTs->has_nameT(text)) { + // if we're inside `f`, replace "T" with TypeDataInt + return substitutedTs->get_substitution_for_nameT(text); + } + if (const Symbol* sym = lookup_global_symbol(text)) { + if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); + } + return custom_type; + } + } + if (TypePtr predefined_type = try_parse_predefined_type(text)) { + return predefined_type; + } + if (treat_unresolved_as_genericT) { + return TypeDataGenericT::create(static_cast(text)); + } + fire_error_unknown_type_name(cur_f, loc, text); + } + + case ast_type_question_nullable: { + TypePtr inner = finalize_type_node(v->as()->get_inner()); + TypePtr result = TypeDataUnion::create_nullable(inner); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + case ast_type_parenthesis_tensor: { + std::vector items = finalize_type_node(v->as()->get_items()); + if (items.size() == 1) { + return items.front(); + } + return TypeDataTensor::create(std::move(items)); + } + + case ast_type_bracket_tuple: { + std::vector items = finalize_type_node(v->as()->get_items()); + return TypeDataBrackets::create(std::move(items)); + } + + case ast_type_arrow_callable: { + std::vector params_and_return = finalize_type_node(v->as()->get_params_and_return()); + TypePtr return_type = params_and_return.back(); + params_and_return.pop_back(); + return TypeDataFunCallable::create(std::move(params_and_return), return_type); + } + + case ast_type_vertical_bar_union: { + std::vector variants = finalize_type_node(v->as()->get_variants()); + TypePtr result = TypeDataUnion::create(std::move(variants)); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + case ast_type_triangle_args: { + const std::vector& inner_and_args = v->as()->get_inner_and_args(); + TypePtr inner = finalize_type_node(inner_and_args.front(), true); + std::vector type_arguments; + type_arguments.reserve(inner_and_args.size() - 1); + for (size_t i = 1; i < inner_and_args.size(); ++i) { + type_arguments.push_back(finalize_type_node(inner_and_args[i])); + } + return instantiate_generic_type_or_fire(cur_f, v->loc, inner, std::move(type_arguments)); + } + + default: + throw UnexpectedASTNodeKind(v, "parse_ast_type_node"); + } + } + + // given `dict` / `User` / `Wrapper` / `WrapperAlias`, find it in a symtable + // for generic types, like `Wrapper`, fire that it's used without type arguments (unless allowed) + // example: `var w: Wrapper = ...`, here will be an error of generic usage without T + // example: `w is Wrapper`, here not, it's allowed (instantiated at type inferring later) + // example: `var w: KKK`, nullptr will be returned + static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcLocation loc, const Symbol* sym, bool allow_without_type_arguments) { + if (AliasDefPtr alias_ref = sym->try_as()) { + if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); + } + if (!visited_aliases.contains(alias_ref)) { + visit_symbol(alias_ref); + } + return TypeDataAlias::create(alias_ref); + } + if (StructPtr struct_ref = sym->try_as()) { + if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); + } + if (!visited_structs.contains(struct_ref)) { + visit_symbol(struct_ref); + } + return TypeDataStruct::create(struct_ref); + } + return nullptr; + } + + // given `Wrapper` / `Pair` / `Response`, instantiate a generic struct/alias + // an error for invalid usage `Pair` / `cell` is also here + static TypePtr instantiate_generic_type_or_fire(FunctionPtr cur_f, SrcLocation loc, TypePtr type_to_instantiate, std::vector&& type_arguments) { + // example: `type WrapperAlias = Wrapper`, we are at `Wrapper`, type_arguments = `` + // they contain generics, so the struct is not ready to be instantiated yet + bool is_still_generic = false; + for (TypePtr argT : type_arguments) { + is_still_generic |= argT->has_genericT_inside(); + } + + if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + if (struct_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(struct_ref, nullptr, std::move(type_arguments)); + } + return TypeDataStruct::create(instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, type_arguments))); + } + if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + AliasDefPtr alias_ref = t_alias->alias_ref; + if (alias_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(nullptr, alias_ref, std::move(type_arguments)); + } + return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); + } + if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { + fire_error_unknown_type_name(cur_f, loc, asT->nameT); + } + // `User` / `cell` + fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); + } + + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { + fire_void_type_not_allowed_inside_union(cur_f, loc, variant); + } + } + } + +public: + + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, bool treat_unresolved_as_genericT) + : cur_f(cur_f) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , treat_unresolved_as_genericT(treat_unresolved_as_genericT) {} + + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { +#ifdef TOLK_DEBUG + tolk_assert(type_node != nullptr); +#endif + TypePtr resolved_type = parse_ast_type_node(type_node, allow_without_type_arguments); + type_node->mutate()->assign_resolved_type(resolved_type); + return resolved_type; + } + + std::vector finalize_type_node(const std::vector& type_node_array) { + std::vector result; + result.reserve(type_node_array.size()); + for (AnyTypeV v : type_node_array) { + result.push_back(finalize_type_node(v)); + } + return result; + } + + static void visit_symbol(GlobalVarPtr glob_ref) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); + TypePtr declared_type = visitor.finalize_type_node(glob_ref->type_node); + glob_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(GlobalConstPtr const_ref) { + if (!const_ref->type_node) { + return; + } + + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); + TypePtr declared_type = visitor.finalize_type_node(const_ref->type_node); + const_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(AliasDefPtr alias_ref) { + static std::vector called_stack; + + // prevent recursion like `type A = B; type B = A` (we can't create TypeDataAlias without a resolved underlying type) + bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); + if (contains) { + throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + } + + if (auto v_genericsT_list = alias_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + alias_ref->mutate()->assign_resolved_genericTs(genericTs); + } + + called_stack.push_back(alias_ref); + TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); + TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); + alias_ref->mutate()->assign_resolved_type(underlying_type); + visited_aliases.insert({alias_ref, 1}); + called_stack.pop_back(); + } + + static void visit_symbol(StructPtr struct_ref) { + visited_structs.insert({struct_ref, 1}); + + if (auto v_genericsT_list = struct_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + struct_ref->mutate()->assign_resolved_genericTs(genericTs); + } + + TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); + field_ref->mutate()->assign_resolved_type(declared_type); + } + } + + static const GenericsDeclaration* construct_genericTs(TypePtr receiver_type, V v_list) { + std::vector itemsT; + if (receiver_type && receiver_type->has_genericT_inside()) { + receiver_type->replace_children_custom([&itemsT](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [asT](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == asT->nameT; + }); + if (it_existing == itemsT.end()) { + itemsT.emplace_back(asT->nameT, nullptr); + } + } + return child; + }); + } + int n_from_receiver = static_cast(itemsT.size()); + + if (v_list) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); + for (int i = 0; i < v_list->size(); ++i) { + auto v_item = v_list->get_item(i); + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == v_item->nameT; + }); + if (it_existing != itemsT.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + TypePtr default_type = nullptr; + if (v_list->get_item(i)->default_type_node) { + default_type = visitor.finalize_type_node(v_list->get_item(i)->default_type_node); + } + itemsT.emplace_back(v_item->nameT, default_type); + } + } + + return new GenericsDeclaration(std::move(itemsT), n_from_receiver); + } +}; + +class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { + TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr, false}; + + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { + return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); + } + +protected: + + void visit(V v) override { + if (v->type_node) { + TypePtr declared_type = finalize_type_node(v->type_node); + v->var_ref->mutate()->assign_resolved_type(declared_type); + } + } + + void visit(V v) override { + tolk_assert(v->sym != nullptr); + + // for `f` / `f`, resolve "MyAlias" and "T" + // (for function call `f()`, this v (ast_reference `f`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + } + + void visit(V v) override { + if (v->pattern_type_node) { + // before `=>` we allow referencing generic types, type inferring will guess + // example: `struct Ok` + `type Response = Ok | ErrCode` + `match (resp) { Ok => ... }` + finalize_type_node(v->pattern_type_node, true); + } + parent::visit(v->get_pattern_expr()); + parent::visit(v->get_body()); + } + + void visit(V v) override { + // for static method calls, like "int.zero()" or "Point.create()", dot obj symbol is unresolved for now + // so, resolve it as a type and store as a "type reference symbol" + if (auto obj_ref = v->get_obj()->try_as()) { + // also, `someFn.prop` doesn't make any sense, show "unknown type"; it also forces `address.staticMethod()` to work + if (obj_ref->sym == nullptr || obj_ref->sym->try_as()) { + std::string_view obj_type_name = obj_ref->get_identifier()->name; + AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); + if (obj_ref->has_instantiationTs()) { // Container.create + std::vector inner_and_args; + inner_and_args.reserve(1 + obj_ref->get_instantiationTs()->size()); + inner_and_args.push_back(obj_type_node); + for (int i = 0; i < obj_ref->get_instantiationTs()->size(); ++i) { + inner_and_args.push_back(obj_ref->get_instantiationTs()->get_item(i)->type_node); + } + obj_type_node = createV(obj_ref->loc, std::move(inner_and_args)); + } + TypePtr type_as_reference = finalize_type_node(obj_type_node); + const Symbol* type_as_symbol = new TypeReferenceUsedAsSymbol(static_cast(obj_type_name), obj_ref->loc, type_as_reference); + obj_ref->mutate()->assign_sym(type_as_symbol); + } + } + + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" + // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + parent::visit(v->get_obj()); + } + + void visit(V v) override { + finalize_type_node(v->type_node); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + finalize_type_node(v->type_node, true); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->type_node) { + finalize_type_node(v->type_node, true); + } + parent::visit(v->get_body()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return !fun_ref->is_builtin_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v) override { + if (fun_ref->receiver_type_node) { + TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); + TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); + std::string name_prefix = receiver_type->as_human_readable(); + bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; + if (embrace) { + name_prefix = "(" + name_prefix + ")"; + } + fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, std::move(name_prefix)); + G.symtable.add_function(fun_ref); + } + if (v->genericsT_list || (fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside())) { + const GenericsDeclaration* genericTs = TypeNodesVisitorResolver::construct_genericTs(fun_ref->receiver_type, v->genericsT_list); + fun_ref->mutate()->assign_resolved_genericTs(genericTs); + } + + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_node(param_ref->type_node); + param_ref->mutate()->assign_resolved_type(declared_type); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } + } + if (fun_ref->return_type_node) { + TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); + fun_ref->mutate()->assign_resolved_type(declared_return_type); + } + + if (fun_ref->is_code_function()) { + parent::visit(v->get_body()->as()); + } + } + + void start_visiting_constant(GlobalConstPtr const_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr, false); + + // `const a = 0 as int8`, resolve types there + // same for struct field `v: int8 = 0 as int8` + parent::visit(const_ref->init_value); + } + + void start_visiting_struct_fields(StructPtr struct_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); + + // same for struct field `v: int8 = 0 as int8` + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } + } +}; + +// prevent recursion like `struct A { field: A }`; +// currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack); +// if there is an annotation to store a struct in a tuple, then it has to be reconsidered; +// it's crucial to detect it here; otherwise, get_width_on_stack() will silently face stack overflow +class InfiniteStructSizeDetector { + static TypePtr visit_type_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataStruct* child_struct = child->try_as()) { + check_struct_for_infinite_size(child_struct->struct_ref); + } + if (const TypeDataAlias* child_alias = child->try_as()) { + return visit_type_deeply(child_alias->underlying_type); + } + return child; + }); + }; + + static void check_struct_for_infinite_size(StructPtr struct_ref) { + static std::vector called_stack; + + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + for (StructFieldPtr field_ref : struct_ref->fields) { + visit_type_deeply(field_ref->declared_type); + } + called_stack.pop_back(); + } + +public: + static void detect_and_fire_if_any_struct_is_infinite() { + for (auto [struct_ref, _] : visited_structs) { + check_struct_for_infinite_size(struct_ref); + } + } +}; + +void pipeline_resolve_types_and_aliases() { + ResolveTypesInsideFunctionVisitor visitor; + + for (const SrcFile* file : G.all_src_files) { + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { + tolk_assert(v_func->fun_ref); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); + } + + } else if (auto v_global = v->try_as()) { + TypeNodesVisitorResolver::visit_symbol(v_global->glob_ref); + + } else if (auto v_const = v->try_as()) { + if (v_const->type_node) { + TypeNodesVisitorResolver::visit_symbol(v_const->const_ref); + } + visitor.start_visiting_constant(v_const->const_ref); + + } else if (auto v_alias = v->try_as()) { + if (!visited_aliases.contains(v_alias->alias_ref)) { + TypeNodesVisitorResolver::visit_symbol(v_alias->alias_ref); + } + + } else if (auto v_struct = v->try_as()) { + if (!visited_structs.contains(v_struct->struct_ref)) { + TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); + } + visitor.start_visiting_struct_fields(v_struct->struct_ref); + } + } + } + + InfiniteStructSizeDetector::detect_and_fire_if_any_struct_is_infinite(); + visited_structs.clear(); + visited_aliases.clear(); + + patch_builtins_after_stdlib_loaded(); +} + +void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { + ResolveTypesInsideFunctionVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +void pipeline_resolve_types_and_aliases(StructPtr struct_ref) { + ResolveTypesInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); + TypeNodesVisitorResolver::visit_symbol(struct_ref); +} + +void pipeline_resolve_types_and_aliases(AliasDefPtr alias_ref) { + TypeNodesVisitorResolver::visit_symbol(alias_ref); +} + +} // namespace tolk diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp new file mode 100644 index 000000000..68aff5060 --- /dev/null +++ b/tolk/pipe-transform-on-message.cpp @@ -0,0 +1,119 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe analyzes the body of + * > fun onInternalMessage(in: InMessage) + * and replaces `in.senderAddress` / etc. with aux AST vertices (handled specially at IR generation). + * + * This function is transformed to + * > fun onInternalMessage(in.body) + * so, + * - accessing `in.body` actually will take an element from a stack + * - accessing `in.senderAddress` will emit `INMSG_SRC` TVM instruction. + */ + +namespace tolk { + +// handle all functions having a prototype `fun f(var: InMessage)` (for testing purposes) +static bool is_onInternalMessage(FunctionPtr fun_ref) { + if (fun_ref->is_entrypoint() || fun_ref->has_tvm_method_id()) { + if (fun_ref->get_num_params() == 1) { + const auto* t_param = fun_ref->get_param(0).declared_type->try_as(); + return t_param && t_param->struct_ref->name == "InMessage"; + } + } + return false; +} + +// `onBouncedMessage` is only one, it's automatically embedded into `onInternalMessage` if exists +static bool is_onBouncedMessage(FunctionPtr fun_ref) { + return fun_ref->is_entrypoint() && fun_ref->name == "onBouncedMessage"; +} + + +struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + LocalVarPtr param_ref = nullptr; // `in` for `fun onInternalMessage(in: InMessage)` + + static void validate_onBouncedMessage(FunctionPtr f) { + if (f->inferred_return_type != TypeDataVoid::create() && f->inferred_return_type != TypeDataNever::create()) { + fire(f, f->loc, "`onBouncedMessage` should return `void`"); + } + if (f->get_num_params() != 1) { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + const auto* t_struct = f->get_param(0).declared_type->try_as(); + if (!t_struct || t_struct->struct_ref->name != "InMessageBounced") { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + } + +protected: + AnyExprV replace(V v) override { + // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) + if (v->sym == param_ref) { + fire(cur_f, v->loc, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); + } + return parent::replace(v); + } + + AnyExprV replace(V v) override { + // replace `in.senderAddress` / `in.valueCoins` with an aux vertex + if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { + if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { + fire(cur_f, v->loc, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); + } + + ASTAuxData* aux_getField = new AuxData_OnInternalMessage_getField(cur_f, v->get_field_name()); + return createV(v->loc, v, aux_getField, v->inferred_type); + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return is_onInternalMessage(fun_ref) || is_onBouncedMessage(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + if (fun_ref->name == "onBouncedMessage") { + validate_onBouncedMessage(fun_ref); + } + + cur_f = fun_ref; + param_ref = &fun_ref->parameters[0]; + + parent::replace(v_function->get_body()); + + std::vector new_parameters; + new_parameters.emplace_back("in.body", fun_ref->loc, TypeDataSlice::create(), nullptr, 0, 0); + fun_ref->mutate()->parameters = std::move(new_parameters); + } +}; + +void pipeline_transform_onInternalMessage() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 0a71d751d..641beb06b 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -34,6 +34,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con void pipeline_register_global_symbols(); void pipeline_resolve_identifiers_and_assign_symbols(); +void pipeline_resolve_types_and_aliases(); void pipeline_calculate_rvalue_lvalue(); void pipeline_infer_types_and_calls_and_fields(); void pipeline_check_inferred_types(); @@ -42,6 +43,10 @@ void pipeline_check_rvalue_lvalue(); void pipeline_check_pure_impure_operations(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); +void pipeline_detect_inline_in_place(); +void pipeline_check_serialized_fields(); +void pipeline_lazy_load_insertions(); +void pipeline_transform_onInternalMessage(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); @@ -49,10 +54,19 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); + void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); +void pipeline_resolve_types_and_aliases(FunctionPtr); void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr); +void pipeline_resolve_types_and_aliases(StructPtr); + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_types_and_aliases(AliasDefPtr); } // namespace tolk diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp new file mode 100644 index 000000000..2b4d80be8 --- /dev/null +++ b/tolk/send-message-api.cpp @@ -0,0 +1,675 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "send-message-api.h" +#include "pack-unpack-serializers.h" +#include "type-system.h" + +namespace tolk { + +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); + +static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { + if (const TypeDataStruct* t_struct = bodyT->unwrap_alias()->try_as()) { + if (t_struct->struct_ref->is_instantiation_of_generic_struct() && t_struct->struct_ref->base_struct_ref->name == "UnsafeBodyNoRef") { + return true; + } + } + return false; +} + +// currently, there is no way to pass custom pack options to createMessage, using hardcoded ones +static std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + tolk_assert(ir_options.size() == 1); + + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + code.emplace_back(loc, Op::_Let, std::vector{ir_options[0]}, std::vector{ir_zero}); // skipBitsNFieldsValidation + return ir_options; +} + +// calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` +static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_shard_depth}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, std::vector{ir_addr_hash}, std::vector{ir_addr_hash, ir_mask[0]}, lookup_function("_&_")); +} + +// struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } +struct IR_AutoDeployAddress { + std::vector is_ContractState, // stateInit is ContractState + is_AddressSharding; // toShard is not null + var_idx_t workchain, // workchain + stateInitCode, stateInitData, stateInitCell, // stateInit + ir_shardDepth, ir_closeTo; // toShard + + IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + tolk_assert(ir_vars.size() == 1 + 3 + 3); + tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); + tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); + + workchain = ir_vars[0]; + + std::vector ir_stateInitUnion(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); + is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitUnion, loc, "(is-ContractState)"); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + stateInitCode = ir_ContractState[0]; + stateInitData = ir_ContractState[1]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + + std::vector ir_toShardOrNull(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); + is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardOrNull, loc, "(is-AddressSharding)"); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + ir_shardDepth = ir_AddressSharding[0]; + ir_closeTo = ir_AddressSharding[1]; + } +}; + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + + const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); + tolk_assert(t_bounce); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); + tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_bounce = next_slice(t_bounce->get_width_on_stack()); + std::vector ir_value = next_slice(t_value->get_width_on_stack()); + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; + // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } + // struct ContractState { code: cell; data: cell; } + // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); + IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); + + // currently, there is no way to pass PackOptions, defaults are used + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 514 (502 CommonMsgInfoRelaxed + 12 StateInit), so 500 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits <= 500 && body_size.max_refs < 2; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // if we need to store body ref, convert it to a cell here, before creating a builder for the message; + // it's more optimal, since the `body` field is the topmost at the stack + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '0' prefix int_msg_info + ctx.storeUint(ir_zero, 1); + // fill `ihr_disabled:Bool` always 1 + ctx.storeUint(ir_one, 1); + // fill `bounce:Bool` from p.bounce (if it's constant (most likely), it will be concatenated with prev and next) + ctx.storeBool(ir_bounce[0]); + // fill `bounced:Bool` + `src:MsgAddress` 00 + ctx.storeUint(ir_zero, 1 + 2); + + // fill `dest:MsgAddressInt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_AutoDeploy = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // input is `dest: { workchain, stateInit, [toShard] }`; + // then calculate hash equal to StateInit cell would be and fill "addr_std$10 + 0 anycast + workchain + hash"; + // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash + code.push_set_cur(if_AutoDeploy.block0); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_dest_ad.workchain, 8); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + { + // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; + // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_dest_ad.ir_shardDepth, ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); + // then hash = (hash of StateInit = 0b00110 (only code + data)) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // input is `dest: { ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_dest_ad.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + { + // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // now, we need: hash = (first D bits from dest.toShard.closeTo) + (last 256-D bits from stateInitHash); + // example for fixedPrefixLength (shard depth) = 8: + // | closeTo | 01010101...xxx | given as input, by user (it's address, internally slice) + // | shardPrefix | 01010101 | first 8 bits of closeTo + // | stateInitHash | yyyyyyyy...yyy | mask = (1 << (256-D)) - 1 = 00000000111...111 (8 zeroes) + // | hash (result) | 01010101...yyy | + // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 + code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_dest_ad.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_dest_ad.ir_shardDepth}, lookup_function("_-_")); + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_dest_ad.ir_closeTo, code.create_int(loc, 3+8, ""), ir_dest_ad.ir_shardDepth }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // input is `dest: { workchain, stateInit }` (toShard is null); + // we already calculated stateInitHash: either cell.hash() or based on code+data + code.push_set_cur(if_sharded.block1); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_AutoDeploy.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: (workchain, hash)` + code.push_set_cur(if_builder.block1); + std::vector ir_dest_wc_hash = transition_to_target_type(std::vector(ir_dest), code, t_dest, t_dest->variants[2], loc); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); + ctx.storeInt(ir_dest_wc_hash[0], 8); // most likely, it's 0 (basechain), will be merged with above + ctx.storeUint(ir_dest_wc_hash[1], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // fill `value:CurrencyCollection` from p.value `coins | (coins, dict)` + std::vector ir_is_coins = pre_compile_is_type(code, t_value, TypeDataCoins::create(), ir_value, loc, "(is-coins)"); + Op& if_coins = code.emplace_back(loc, Op::_If, ir_is_coins); + { + code.push_set_cur(if_coins.block0); + std::vector ir_coins = transition_to_target_type(std::vector(ir_value), code, t_value, TypeDataCoins::create(), loc); + ctx.storeCoins(ir_coins[0]); + ctx.storeUint(ir_zero, 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_coins.block1); + std::vector ir_coins_dict = transition_to_target_type(std::move(ir_value), code, t_value, t_value->variants[1], loc); + ctx.storeCoins(ir_coins_dict[0]); + ctx.storeMaybeRef(ir_coins_dict[1]); + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 4*0 ihr_fee + 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at + ctx.storeUint(ir_zero, 4 + 4 + 64 + 32); + + // fill `init: (Maybe (Either StateInit ^StateInit))` + // it's present only if p.dest contains StateInit + // also fill the either bit of `body: (Either X ^X)` + Op& if_no_init = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // when it's known at compile-time (always in practice), this `if` is simplified, and bits join with above + code.push_set_cur(if_no_init.block1); + ctx.storeUint(body_store_as_ref ? ir_one : ir_zero, 1 + 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_no_init.block0); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; + // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); + { + // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not + code.push_set_cur(if_sharded.block0); + ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); + ctx.storeUint(ir_dest_ad.ir_shardDepth, 5); // fixedPrefixLength (shard depth) + ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); + code.close_pop_cur(loc); + // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) + } + { + // 1 (maybe true) + 0 (either left) + 00110 (only code and data from StateInit) + body ref or not + code.push_set_cur(if_sharded.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b10001100 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); + code.close_pop_cur(loc); + } + ctx.storeRef(ir_dest_ad.stateInitCode); + ctx.storeRef(ir_dest_ad.stateInitData); + code.close_pop_cur(loc); + } + { + // so, we have `dest: { stateInit: someCell }`, store it as ref + // 1 (maybe true) + 1 (either right) + body ref or not + code.push_set_cur(if_ContractState.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 1); + ctx.storeRef(ir_dest_ad.stateInitCell); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // store body; previously, we've calculated whether to store is as a ref or not + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + tolk_assert(ir_body.empty()); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); // it was either an input cell or a automatically created one + ctx.storeRef(ir_body[0]); + } else { + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); + StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); + + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_topic = s_ExtOutLogBucket->find_field("topic")->declared_type->try_as(); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (2+1) && t_dest->size() == 3); + tolk_assert(t_topic && t_topic->get_width_on_stack() == (1+1) && t_topic->size() == 2); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | builder | ExtOutLogBucket`; + // struct ExtOutLogBucket { topic: uint248 | bits248; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), loc); + // dest.topic (it's the only field in a struct) + std::vector ir_dest_topic = std::move(ir_dest_ExtOutLogBucket); + + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 622 (621 CommonMsgInfoRelaxed + 1 StateInit), so 400 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits < 400; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // same as for createMessage: `body` field is the topmost at the stack, convert it to a cell before creating a builder + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '11' prefix ext_out_msg_info + '00' src + ctx.storeUint(code.create_int(loc, 0b1100, "(out-prefix)"), 4); + + // fill `dest:MsgAddressExt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket`; + // fill addr_extern$01 + 256 (len 9 bit) + 0x00 (prefix) + 248 bits + code.push_set_cur(if_builder.block1); + ctx.storeUint(ir_one, 2); // addr_extern$01 + ctx.storeUint(code.create_int(loc, 256, "(addr-len)"), 9); // len:(## 9) = 256 + ctx.storeOpcode(s_ExtOutLogBucket->opcode); + std::vector ir_if_topic_uint = pre_compile_is_type(code, t_topic, t_topic->variants[0], ir_dest_topic, loc, "(topic-is-uint)"); + Op& if_topic_uint = code.emplace_back(loc, Op::_If, ir_if_topic_uint); + { + // input is `dest: ExtOutLogBucket { topic: uint248 }` + code.push_set_cur(if_topic_uint.block0); + ctx.storeUint(ir_dest_topic[0], 248); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket { topic: bits248 }` + // for this field, generate runtime check to ensure its length + code.push_set_cur(if_topic_uint.block1); + ctx.generate_pack_any(t_topic->variants[1], std::vector{ir_dest_topic[0]}); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 64*0 created_lt + 32*0 created_at + // plus, StateInit is empty (0 maybe bit) for external messages + ctx.storeUint(ir_zero, 64 + 32 + 1); + + // fill bit `body: (Either X ^X)` and store body + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + // missing body of type `never` + tolk_assert(ir_body.empty()); + ctx.storeUint(ir_zero, 1); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); + ctx.storeUint(ir_one, 1); + ctx.storeRef(ir_body[0]); + } else { + ctx.storeUint(ir_zero, 1); + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options) { + tolk_assert(ir_shard_options.size() == 2); + + // example for fixedPrefixLength (shard depth) = 8: + // | self (A) | aaaaaaaaaaa...aaa | + // | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) + // | result | 01010101aaa...aaa | address of A in same shard as B + + // the most effective way is not to calculate shardPrefix, but to: + // - take first 3+8+D bits of B: we'll have '100' (std addr no anycast) + workchainB + shardPrefix + // - take last 256-D bits of A: we'll have "aa...a" + // - concatenate: we'll result in '100' + workchainB + "bbaa...a" + + std::vector ir_offsetB = {code.create_int(loc, 3 + 8, "(offset-addrB)")}; + code.emplace_back(loc, Op::_Call, ir_offsetB, std::vector{ir_offsetB[0], ir_shard_options[0]}, lookup_function("_+_")); + std::vector ir_headB = code.create_tmp_var(TypeDataSlice::create(), loc, "(headB)"); + code.emplace_back(loc, Op::_Call, ir_headB, std::vector{ir_shard_options[1], ir_offsetB[0]}, lookup_function("slice.getFirstBits")); + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_headB[0]}, lookup_function("builder.storeSlice")); + + std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; + code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); + std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); + code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{ir_self_address[0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); + + return ir_builder; +} + +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); + // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell + // (for fewer stack manipulations) + + // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // keep hash = (last 256-D bits from stateInitHash) = `hash & mask` + code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_self.ir_shardDepth}, lookup_function("_-_")); + + // calculate shard_prefix = (first D bits from dest.toShard.closeTo) + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_self.ir_closeTo, code.create_int(loc, 3+8, ""), ir_self.ir_shardDepth }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + + // on a stack: stateInitHash & mask; shard prefix; create a cell and store all + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // called `{ workchain, stateInit }` (toShard is null); + // on a stack: hash (already calculated); create a cell and store all + code.push_set_cur(if_sharded.block1); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + + return ir_builder; +} + +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` + Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded1.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded1.block1); + code.close_pop_cur(loc); + } + + // now do `(wc, hash) = addr.getWorkchainAndHash()` + std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_address, lookup_function("address.getWorkchainAndHash")); + + // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) + Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded2.block0); + append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded2.block1); + code.close_pop_cur(loc); + } + + // finally, eval `(hash == stateInitHash) & (wc == workchain)` + std::vector ir_eq_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-hash)"); + code.emplace_back(loc, Op::_Call, ir_eq_hash, std::vector{ir_addr_wc_hash[1], ir_hash[0]}, lookup_function("_==_")); + std::vector ir_eq_wc = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-wc)"); + code.emplace_back(loc, Op::_Call, ir_eq_wc, std::vector{ir_addr_wc_hash[0], ir_self.workchain}, lookup_function("_==_")); + + std::vector ir_bool_result = code.create_tmp_var(TypeDataBool::create(), loc, "(is-addr-result)"); + code.emplace_back(loc, Op::_Call, ir_bool_result, std::vector{ir_eq_hash[0], ir_eq_wc[0]}, lookup_function("_&_")); + return ir_bool_result; +} + +} // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h new file mode 100644 index 000000000..f86be5446 --- /dev/null +++ b/tolk/send-message-api.h @@ -0,0 +1,31 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "tolk.h" + +namespace tolk { + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); + +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy); +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address); + +} // namespace tolk diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 7b86f519f..7dd60ee97 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -98,14 +98,32 @@ namespace tolk { std::string SinkExpression::to_string() const { std::string result = var_ref->name; uint64_t cur_path = index_path; + TypePtr cur_type = var_ref->declared_type; while (cur_path != 0) { result += "."; - result += std::to_string((cur_path & 0xFF) - 1); + if (const TypeDataStruct* t_struct = cur_type->try_as()) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field((cur_path & 0xFF) - 1); + result += field_ref->name; + cur_type = field_ref->declared_type; + } else { + result += std::to_string((cur_path & 0xFF) - 1); + } cur_path >>= 8; } return result; } +SinkExpression SinkExpression::get_child_s_expr(int field_idx) const { + uint64_t new_index_path = index_path; // if we have c.1 (index_path = 2) and construct c.1.N, calc (N<<8 + 2) + for (int empty_byte = 0; empty_byte < 8; ++empty_byte) { + if ((index_path & (0xFF << (empty_byte*8))) == 0) { + new_index_path += (field_idx + 1) << (empty_byte*8); + break; + } + } + return SinkExpression(var_ref, new_index_path); +} + static std::string to_string(SignState s) { static const char* txt[6 + 1] = {"sign=unknown", ">0", "<0", "=0", ">=0", "<=0", "sign=never"}; return txt[static_cast(s)]; @@ -133,10 +151,15 @@ static AnyExprV unwrap_not_null_operator(AnyExprV expr) { // 3) when two data flows rejoin // example: `if (tensorVar != null) ... else ...` rejoin `(int,int)` and `null` into `(int,int)?` // when lca can't be calculated (example: `(int,int)` and `(int,int,int)`), nullptr is returned -static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { - if (a == b) { +static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nullptr) { + if (a->equal_to(b)) { return a; } + + if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { + return TypeDataUnknown::create(); + } + if (a == TypeDataNever::create()) { return b; } @@ -144,22 +167,11 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return a; } - if (a->can_rhs_be_assigned(b)) { - return a; - } - if (b->can_rhs_be_assigned(a)) { - return b; - } - - if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { - return TypeDataUnknown::create(); - } - if (a == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(b); + return TypeDataUnion::create_nullable(b); } if (b == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(a); + return TypeDataUnion::create_nullable(a); } const auto* tensor1 = a->try_as(); @@ -168,7 +180,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { std::vector types_lca; types_lca.reserve(tensor1->size()); for (int i = 0; i < tensor1->size(); ++i) { - TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i]); + TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i], became_union); if (next == nullptr) { return nullptr; } @@ -177,22 +189,49 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataTensor::create(std::move(types_lca)); } - const auto* tuple1 = a->try_as(); - const auto* tuple2 = b->try_as(); + const auto* tuple1 = a->try_as(); + const auto* tuple2 = b->try_as(); if (tuple1 && tuple2 && tuple1->size() == tuple2->size()) { std::vector types_lca; types_lca.reserve(tuple1->size()); for (int i = 0; i < tuple1->size(); ++i) { - TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i]); + TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i], became_union); if (next == nullptr) { return nullptr; } types_lca.push_back(next); } - return TypeDataTypedTuple::create(std::move(types_lca)); + return TypeDataBrackets::create(std::move(types_lca)); + } + + if (const auto* a_alias = a->try_as()) { + return calculate_type_lca(a_alias->underlying_type, b, became_union); + } + if (const auto* b_alias = b->try_as()) { + return calculate_type_lca(a, b_alias->underlying_type, became_union); } - return nullptr; + TypePtr resulting_union = TypeDataUnion::create(std::vector{a, b}); + if (became_union != nullptr && !a->equal_to(resulting_union) && !b->equal_to(resulting_union)) { + *became_union = true; + } + return resulting_union; +} + +// when `var v = rhs`, `v` is `unknown` before assignment (before rhs->inferred_type is assigned to it); +// when `var (v1,v2,v3) = rhs`, left side is `(unknown,unknown,unknown)` +static bool is_type_unknown_from_var_lhs_decl(TypePtr t) { + if (t == TypeDataUnknown::create()) { + return true; + } + if (const auto* t_tensor = t->try_as()) { + bool all_unknown = true; + for (TypePtr item : t_tensor->items) { + all_unknown &=is_type_unknown_from_var_lhs_decl(item); + } + return all_unknown; + } + return false; } // merge (unify) of two sign states: what sign do we definitely have @@ -237,31 +276,28 @@ BoolState calculate_bool_lca(BoolState a, BoolState b) { // see comments above TypeInferringUnifyStrategy // this function calculates lca or currently stored result and next -bool TypeInferringUnifyStrategy::unify_with(TypePtr next) { +void TypeInferringUnifyStrategy::unify_with(TypePtr next, TypePtr dest_hint) { + // example: `var r = ... ? int8 : int16`, will be inferred as `int8 | int16` (via unification) + // but `var r: int = ... ? int8 : int16`, will be inferred as `int` (it's dest_hint) + if (dest_hint && !is_type_unknown_from_var_lhs_decl(dest_hint) && !dest_hint->unwrap_alias()->try_as()) { + if (dest_hint->can_rhs_be_assigned(next)) { + next = dest_hint; + } + } + if (unified_result == nullptr) { unified_result = next; - return true; + return; } if (unified_result == next) { - return true; + return; } - TypePtr combined = calculate_type_lca(unified_result, next); - if (!combined) { - return false; - } + bool became_union = false; + TypePtr combined = calculate_type_lca(unified_result, next, &became_union); + different_types_became_union |= became_union; unified_result = combined; - return true; -} - -bool TypeInferringUnifyStrategy::unify_with_implicit_return_void() { - if (unified_result == nullptr) { - unified_result = TypeDataVoid::create(); - return true; - } - - return unified_result == TypeDataVoid::create(); } // invalidate knowledge about sub-fields of a variable or its field @@ -345,14 +381,43 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { return FlowContext(std::move(unified), c1.unreachable && c2.unreachable); } -// return `T`, so that `T?` = type -// what for: `if (x != null)`, to smart cast x inside if -TypePtr calculate_type_subtract_null(TypePtr type) { - if (const auto* as_nullable = type->try_as()) { - return as_nullable->inner; +// return `T`, so that `T + subtract_type` = type +// example: `int?` - `null` = `int` +// example: `int | slice | builder | bool` - `bool | slice` = `int | builder` +// what for: `if (x != null)` / `if (x is T)`, to smart cast x inside if +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { + const TypeDataUnion* lhs_union = type->try_as(); + if (!lhs_union) { + return TypeDataNever::create(); + } + + std::vector rest_variants; + + if (const TypeDataUnion* sub_union = subtract_type->try_as()) { + if (lhs_union->has_all_variants_of(sub_union)) { + rest_variants.reserve(lhs_union->size() - sub_union->size()); + for (TypePtr lhs_variant : lhs_union->variants) { + if (!sub_union->has_variant_with_type_id(lhs_variant)) { + rest_variants.push_back(lhs_variant); + } + } + } + } else if (lhs_union->has_variant_with_type_id(subtract_type)) { + rest_variants.reserve(lhs_union->size() - 1); + for (TypePtr lhs_variant : lhs_union->variants) { + if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { + rest_variants.push_back(lhs_variant); + } + } + } + + if (rest_variants.empty()) { + return TypeDataNever::create(); + } + if (rest_variants.size() == 1) { + return rest_variants[0]; } - // union types will be handled here - return TypeDataNever::create(); + return TypeDataUnion::create(std::move(rest_variants)); } // given any expression vertex, extract SinkExpression is possible @@ -369,11 +434,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + if (auto as_dot = v->try_as()) { V cur_dot = as_dot; uint64_t index_path = 0; - while (cur_dot->is_target_indexed_access()) { - int index_at = std::get(cur_dot->target); + while (cur_dot->is_target_indexed_access() || cur_dot->is_target_struct_field()) { + int index_at = cur_dot->is_target_indexed_access() + ? std::get(cur_dot->target) + : std::get(cur_dot->target)->field_idx; index_path = (index_path << 8) + index_at + 1; if (auto parent_dot = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { cur_dot = parent_dot; @@ -382,7 +449,7 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } if (auto as_ref = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { - if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && index_path) { return SinkExpression(var_ref, index_path); } } @@ -396,6 +463,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { return extract_sink_expression_from_vertex(as_assign->get_lhs()); } + if (auto as_decl = v->try_as()) { + if (auto decl_var = as_decl->get_expr()->try_as()) { + tolk_assert(decl_var->var_ref); + return SinkExpression(decl_var->var_ref); + } + } + return {}; } @@ -410,14 +484,20 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { - TypePtr obj_type = as_dot->get_obj()->inferred_type; // v already inferred; hence, index_at is correct - int index_at = std::get(as_dot->target); - if (const auto* t_tensor = obj_type->try_as()) { - return t_tensor->items[index_at]; + if (auto as_dot = v->try_as()) { + TypePtr obj_type = as_dot->get_obj()->inferred_type->unwrap_alias(); // v already inferred; hence, index_at is correct + if (as_dot->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(as_dot->target); + return field_ref->declared_type; } - if (const auto* t_tuple = obj_type->try_as()) { - return t_tuple->items[index_at]; + if (as_dot->is_target_indexed_access()) { + int index_at = std::get(as_dot->target); + if (const auto* t_tensor = obj_type->try_as()) { + return t_tensor->items[index_at]; + } + if (const auto* t_tuple = obj_type->try_as()) { + return t_tuple->items[index_at]; + } } } @@ -430,18 +510,32 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { // obvious example: `var x: (int,int)? = null`, it's `null` (`x == null` is always true, `x` can be passed to any `T?`) // not obvious example: `var x: (int?, int?)? = (3,null)`, result is `(int?,int?)`, whereas type of rhs is `(int,null)` TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { - // assign `T` to `T?` (or at least "assignable-to-T" to "T?") - // smart cast to `T` - if (const auto* lhs_nullable = lhs_declared_type->try_as()) { - if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { - return lhs_nullable->inner; + if (const TypeDataUnion* lhs_union = lhs_declared_type->unwrap_alias()->try_as()) { + // example: `var x: T? = null`, result is null + // example: `var x: int | (int, User?) = (5, null)`, result is `(int, User?)` + if (TypePtr lhs_subtype = lhs_union->calculate_exact_variant_to_fit_rhs(rhs_inferred_type)) { + return lhs_subtype; + } + // example: `var x: int | slice | cell = 4`, result is int + // example: `var x: T1 | T2 | T3 = y as T3 | T1`, result is `T1 | T3` + if (const TypeDataUnion* rhs_union = rhs_inferred_type->try_as()) { + bool lhs_has_all_variants_of_rhs = true; + for (TypePtr rhs_variant : rhs_union->variants) { + lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); + } + if (lhs_has_all_variants_of_rhs && rhs_union->size() < lhs_union->size()) { + std::vector subtypes_of_lhs; + for (TypePtr lhs_variant : lhs_union->variants) { + if (rhs_union->has_variant_with_type_id(lhs_variant)) { + subtypes_of_lhs.push_back(lhs_variant); + } + } + if (subtypes_of_lhs.size() == 1) { + return subtypes_of_lhs[0]; + } + return TypeDataUnion::create(std::move(subtypes_of_lhs)); + } } - } - - // assign `null` to `T?` - // smart cast to `null` - if (lhs_declared_type->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { - return TypeDataNullLiteral::create(); } // no smart cast, type is the same as declared diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index b97c8864c..b579e8f31 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -18,6 +18,7 @@ #include "fwd-declarations.h" #include "type-system.h" +#include #include #include @@ -34,12 +35,13 @@ namespace tolk { */ class TypeInferringUnifyStrategy { TypePtr unified_result = nullptr; + bool different_types_became_union = false; public: - bool unify_with(TypePtr next); - bool unify_with_implicit_return_void(); + void unify_with(TypePtr next, TypePtr dest_hint = nullptr); TypePtr get_result() const { return unified_result; } + bool is_union_of_different_types() const { return different_types_became_union; } }; /* @@ -69,6 +71,7 @@ struct SinkExpression { explicit operator bool() const { return var_ref != nullptr; } std::string to_string() const; + SinkExpression get_child_s_expr(int field_idx) const; }; // UnreachableKind is a reason of why control flow is unreachable or interrupted @@ -200,7 +203,7 @@ struct ExprFlow { std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts); std::ostream& operator<<(std::ostream& os, const FlowContext& flow); -TypePtr calculate_type_subtract_null(TypePtr type); +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type); SinkExpression extract_sink_expression_from_vertex(AnyExprV v); TypePtr calc_declared_type_before_smart_cast(AnyExprV v); TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type); diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 1286c1f9c..07db0eb88 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -24,66 +24,57 @@ namespace tolk { static_assert(sizeof(SrcLocation) == 8); -const SrcFile* AllRegisteredSrcFiles::find_file(int file_id) const { +const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) const { + // files with the same realpath are considered equal for (const SrcFile* file : all_src_files) { - if (file->file_id == file_id) { + if (file->realpath == realpath) { return file; } } return nullptr; } -const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { - for (const SrcFile* file : all_src_files) { - if (file->abs_filename == abs_filename) { - return file; - } - } - return nullptr; -} +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, SrcLocation included_from) { + bool is_stdlib = filename.size() > 8 && filename.starts_with("@stdlib/"); -const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) { - td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str()); + td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str()); if (path.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); } - throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str()); + throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str()); } - std::string abs_filename = path.move_as_ok(); - if (const SrcFile* file = find_file(abs_filename)) { + std::string realpath = path.move_as_ok(); + if (const SrcFile* file = find_file(realpath)) { return file; } - td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str()); + td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str()); if (text.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); } - throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); + throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str()); } - SrcFile* created = new SrcFile(++last_registered_file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + int file_id = static_cast(all_src_files.size()); // SrcFile::file_id is the index in all files + SrcFile* created = new SrcFile(file_id, is_stdlib, std::move(realpath), text.move_as_ok()); if (G.is_verbosity(1)) { - std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; + std::cerr << "register file_id " << created->file_id << " " << created->realpath << std::endl; } all_src_files.push_back(created); return created; } SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { + int last_registered_file_id = static_cast(all_src_files.size() - 1); if (last_parsed_file_id >= last_registered_file_id) { return nullptr; } return const_cast(all_src_files[++last_parsed_file_id]); } -bool SrcFile::is_stdlib_file() const { - std::string_view rel(rel_filename); - return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc -} - bool SrcFile::is_offset_valid(int offset) const { return offset >= 0 && offset < static_cast(text.size()); } @@ -93,6 +84,10 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, -1, -1, "invalid offset"}; } + // currently, converting offset to line number is O(N): just read file contents char by char and detect lines + // since original Tolk src lines are now printed into Fift output, this is invoked for every asm instruction + // but anyway, it consumes a small amount of time relative to other work of the compiler + // in the future, it can be optimized by making lines index aside just std::string_view text int line_idx = 0; int char_idx = 0; int line_offset = 0; @@ -119,9 +114,30 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; } +std::string SrcFile::extract_short_name() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return realpath; + } + std::string short_name = realpath.substr(last_slash + 1); // "file.tolk" (no path) + + if (is_stdlib_file) { // not "common.tolk", but "@stdlib/common" + return "@stdlib/" + short_name.substr(0, short_name.size() - 5); + } + return short_name; +} + +std::string SrcFile::extract_dirname() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return ""; + } + return realpath.substr(0, last_slash + 1); +} + std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { - return os << (src_file ? src_file->rel_filename : "unknown-location"); + return os << (src_file ? src_file->realpath : "unknown-location"); } std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { @@ -129,7 +145,7 @@ std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { } const SrcFile* SrcLocation::get_src_file() const { - return G.all_src_files.find_file(file_id); + return G.all_src_files.get_file(file_id); } void SrcLocation::show(std::ostream& os) const { @@ -137,7 +153,10 @@ void SrcLocation::show(std::ostream& os) const { os << src_file; if (src_file && src_file->is_offset_valid(char_offset)) { SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); - os << ':' << pos.line_no << ':' << pos.char_no; + os << ':' << pos.line_no; + if (pos.char_no != 1) { + os << ':' << pos.char_no; + } } } @@ -157,6 +176,37 @@ void SrcLocation::show_context(std::ostream& os) const { os << '^' << "\n"; } +// when generating Fift output, every block of asm instructions originated from the same Tolk line, +// is preceded by an original line as a comment +void SrcLocation::show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const { + SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(char_offset); + + // avoid duplicating one line multiple times in fift output + if (pos.line_no == *last_line_no) { + return; + } + *last_line_no = pos.line_no; + + // trim some characters from start and end to see `else if (x)` not `} else if (x) {` + std::string_view s = pos.line_str; + int b = 0, e = static_cast(s.size() - 1); + while (std::isspace(s[b]) || s[b] == '}') { + if (b < e) b++; + else break; + } + while (std::isspace(s[e]) || s[e] == '{' || s[e] == ';' || s[e] == ',') { + if (e > b) e--; + else break; + } + + if (b < e) { + for (int i = 0; i < indent * 2; ++i) { + os << ' '; + } + os << "// " << pos.line_no << ": " << s.substr(b, e - b + 1) << std::endl; + } +} + std::string SrcLocation::to_string() const { std::ostringstream os; show(os); @@ -195,9 +245,29 @@ std::ostream& operator<<(std::ostream& os, const ParseError& error) { } void ParseError::show(std::ostream& os) const { - os << loc << ": error: " << message << std::endl; + if (message.find('\n') == std::string::npos) { + // just print a single-line message + os << loc << ": error: " << message << std::endl; + } else { + // print "location: line1 \n (spaces) line2 \n ..." + std::string_view message = this->message; + std::string loc_text = loc.to_string(); + std::string loc_spaces(std::min(static_cast(loc_text.size()), 9), ' '); + size_t start = 0, end; + os << loc_text << ": error: "; + while ((end = message.find('\n', start)) != std::string::npos) { + if (start > 0) { + os << loc_spaces << " "; + } + os << message.substr(start, end - start) << std::endl; + start = end + 1; + } + if (start < message.size()) { + os << loc_spaces << " " << message.substr(start) << std::endl; + } + } if (current_function) { - os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + os << std::endl << " // in function `" << current_function->as_human_readable() << "`" << std::endl; } loc.show_context(os); } diff --git a/tolk/src-file.h b/tolk/src-file.h index b0f9cba37..9b39a0acc 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -35,25 +35,26 @@ struct SrcFile { }; int file_id; // an incremental counter through all parsed files - std::string rel_filename; // relative to cwd - std::string abs_filename; // absolute from root + bool is_stdlib_file; // is a part of Tolk distribution, imported via "@stdlib/..." + std::string realpath; // what "realpath" returned to locate (either abs path or what tolk-js returns) std::string text; // file contents loaded into memory, every Token::str_val points inside it AnyV ast = nullptr; // when a file has been parsed, its ast_tolk_file is kept here std::vector imports; // to check strictness (can't use a symbol without importing its file) - SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text) + SrcFile(int file_id, bool is_stdlib_file, std::string realpath, std::string&& text) : file_id(file_id) - , rel_filename(std::move(rel_filename)) - , abs_filename(std::move(abs_filename)) + , is_stdlib_file(is_stdlib_file) + , realpath(std::move(realpath)) , text(std::move(text)) { } SrcFile(const SrcFile& other) = delete; SrcFile &operator=(const SrcFile&) = delete; - bool is_stdlib_file() const; - bool is_offset_valid(int offset) const; SrcPosition convert_offset(int offset) const; + + std::string extract_short_name() const; + std::string extract_dirname() const; }; @@ -85,6 +86,7 @@ class SrcLocation { void show(std::ostream& os) const; void show_context(std::ostream& os) const; + void show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const; std::string to_string() const; void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; @@ -97,14 +99,13 @@ std::ostream& operator<<(std::ostream& os, SrcLocation loc); class AllRegisteredSrcFiles { std::vector all_src_files; - int last_registered_file_id = -1; int last_parsed_file_id = -1; public: - const SrcFile* find_file(int file_id) const; - const SrcFile* find_file(const std::string& abs_filename) const; + const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } + const SrcFile* find_file(const std::string& realpath) const; - const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); + const SrcFile* locate_and_register_source_file(const std::string& filename, SrcLocation included_from); SrcFile* get_next_unparsed_file(); auto begin() const { return all_src_files.begin(); } diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp index fe5735e5c..d86afaec6 100644 --- a/tolk/stack-transform.cpp +++ b/tolk/stack-transform.cpp @@ -254,7 +254,7 @@ bool StackTransform::compose(const StackTransform &a, const StackTransform &b, S x1 = a.try_load(i); } int y = b.A[j - 1].second; - if (!c.try_store(x2, a(y))) { + if (!c.try_store(x2, a.get(y))) { return false; } x2 = b.try_load(j, a.d); @@ -360,18 +360,6 @@ StackTransform StackTransform::Xchg(int i, int j, bool relaxed) { return t; } -StackTransform StackTransform::Push(int i) { - StackTransform t; - t.apply_push(i); - return t; -} - -StackTransform StackTransform::Pop(int i) { - StackTransform t; - t.apply_pop(i); - return t; -} - bool StackTransform::is_xchg(int i, int j) const { if (i == j) { return is_id(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 51dc34401..348fac839 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -21,13 +21,49 @@ namespace tolk { +void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const { + const SrcFile* declared_in = loc.get_src_file(); + bool has_import = false; + for (const SrcFile::ImportDirective& import : used_loc.get_src_file()->imports) { + if (import.imported_file == declared_in) { + has_import = true; + } + } + if (!has_import) { + throw ParseError(cur_f, used_loc, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); + } +} + std::string FunctionData::as_human_readable() const { - if (!genericTs) { + if (!is_generic_function()) { return name; // if it's generic instantiation like `f`, its name is "f", not "f" } return name + genericTs->as_human_readable(); } +std::string AliasDefData::as_human_readable() const { + if (!is_generic_alias()) { + return name; + } + return name + genericTs->as_human_readable(); +} + +std::string StructData::as_human_readable() const { + if (!is_generic_struct()) { + return name; + } + return name + genericTs->as_human_readable(); +} + +LocalVarPtr FunctionData::find_param(std::string_view name) const { + for (const LocalVarData& param_data : parameters) { + if (param_data.name == name) { + return ¶m_data; + } + } + return nullptr; +} + bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body if (!is_really_used() && G.settings.remove_unused_functions) { @@ -46,12 +82,30 @@ bool FunctionData::does_need_codegen() const { if (is_generic_function()) { return false; } + // if calls to this function were inlined in place, the function itself is omitted from fif + if (is_inlined_in_place()) { + return false; + } // currently, there is no inlining, all functions are codegenerated // (but actually, unused ones are later removed by Fift) // in the future, we may want to implement a true AST inlining for "simple" functions return true; } +void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix) { + this->receiver_type = receiver_type; + if (!this->substitutedTs) { // after receiver has been resolve, update name to "receiver.method" + name_prefix.erase(std::remove(name_prefix.begin(), name_prefix.end(), ' '), name_prefix.end()); + this->name = name_prefix + "." + this->method_name; + } +} + +void FunctionData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + void FunctionData::assign_resolved_type(TypePtr declared_return_type) { this->declared_return_type = declared_return_type; } @@ -77,6 +131,10 @@ void FunctionData::assign_is_really_used() { this->flags |= flagReallyUsed; } +void FunctionData::assign_inline_mode_in_place() { + this->inline_mode = FunctionInlineMode::inlineInPlace; +} + void FunctionData::assign_arg_order(std::vector&& arg_order) { this->arg_order = std::move(arg_order); } @@ -93,6 +151,18 @@ void GlobalConstData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } +void GlobalConstData::assign_inferred_type(TypePtr inferred_type) { + this->inferred_type = inferred_type; +} + +void GlobalConstData::assign_init_value(AnyExprV init_value) { + this->init_value = init_value; +} + +void LocalVarData::assign_used_as_lval() { + this->flags |= flagUsedAsLVal; +} + void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { this->ir_idx = std::move(ir_idx); } @@ -105,6 +175,61 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void LocalVarData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + +void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + +void AliasDefData::assign_resolved_type(TypePtr underlying_type) { + this->underlying_type = underlying_type; +} + +void StructFieldData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void StructFieldData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + +void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + +StructFieldPtr StructData::find_field(std::string_view field_name) const { + for (StructFieldPtr field : fields) { + if (field->name == field_name) { + return field; + } + } + return nullptr; +} + +// formats opcode as "x{...}" or "b{...}" +std::string StructData::PackOpcode::format_as_slice() const { + const int base = prefix_len % 4 == 0 ? 16 : 2; + const int s_len = base == 16 ? prefix_len / 4 : prefix_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + int64_t opcode = pack_prefix; + for (int i = s_len - 1; i >= 0 && opcode != 0; --i) { + result[2 + i] = digits[opcode % base]; + opcode /= base; + } + return result; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -141,8 +266,44 @@ void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { } } +void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { + auto key = key_hash(a_sym->name); + auto [it, inserted] = entries.emplace(key, a_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(a_sym->loc, it->second); + } +} + +void GlobalSymbolTable::add_struct(StructPtr s_sym) { + auto key = key_hash(s_sym->name); + auto [it, inserted] = entries.emplace(key, s_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(s_sym->loc, it->second); + } +} + +void GlobalSymbolTable::replace_function(FunctionPtr f_sym) { + auto key = key_hash(f_sym->name); + assert(entries.contains(key)); + entries[key] = f_sym; +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } +FunctionPtr lookup_function(std::string_view name) { + return G.symtable.lookup(name)->try_as(); +} + +std::vector lookup_methods_with_name(std::string_view name) { + std::vector result; + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == name) { + result.push_back(method_ref); + } + } + return result; +} + } // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index 9419afce6..268b474d3 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -18,7 +18,6 @@ #include "src-file.h" #include "fwd-declarations.h" -#include "constant-evaluator.h" #include "crypto/common/refint.h" #include #include @@ -44,22 +43,37 @@ struct Symbol { #endif return dynamic_cast(this); } + + void check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const; }; struct LocalVarData final : Symbol { enum { flagMutateParameter = 1, // parameter was declared with `mutate` keyword flagImmutable = 2, // variable was declared via `val` (not `var`) + flagLateInit = 4, // variable was declared via `lateinit` (not assigned at declaration) + flagUsedAsLVal = 8, // variable is assigned or in another way used as lvalue inside a function }; - TypePtr declared_type; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value = nullptr; // for function parameters, if it has a default value int flags; int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; - LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) + LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV default_value, int flags, int param_idx) + : Symbol(std::move(name), loc) + , type_node(type_node) + , default_value(default_value) + , flags(flags) + , param_idx(param_idx) { + } + LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV default_value, int flags, int param_idx) : Symbol(std::move(name), loc) + , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) + , default_value(default_value) , flags(flags) , param_idx(param_idx) { } @@ -67,19 +81,23 @@ struct LocalVarData final : Symbol { bool is_parameter() const { return param_idx >= 0; } bool is_immutable() const { return flags & flagImmutable; } + bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } + bool is_used_as_lval() const { return flags & flagUsedAsLVal; } + bool has_default_value() const { return default_value != nullptr; } LocalVarData* mutate() const { return const_cast(this); } + void assign_used_as_lval(); void assign_ir_idx(std::vector&& ir_idx); void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); + void assign_default_value(AnyExprV default_value); }; struct FunctionBodyCode; struct FunctionBodyAsm; struct FunctionBodyBuiltin; struct GenericsDeclaration; -struct GenericsInstantiation; typedef std::variant< FunctionBodyCode*, @@ -88,44 +106,72 @@ typedef std::variant< > FunctionBody; struct FunctionData final : Symbol { - static constexpr int EMPTY_METHOD_ID = -10; + static constexpr int EMPTY_TVM_METHOD_ID = -10; enum { - flagInline = 1, // marked `@inline` - flagInlineRef = 2, // marked `@inline_ref` flagTypeInferringDone = 4, // type inferring step of function's body (all AST nodes assigning v->inferred_type) is done flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end - flagGetMethod = 64, // was declared via `get func(): T`, method_id is auto-assigned - flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. + flagContractGetter = 64, // was declared via `get func(): T`, tvm_method_id is auto-assigned + flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. flagHasMutateParams = 256, // has parameters declared as `mutate` flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated + flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others + flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. + flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type + flagManualOnBounce = 32768, // for onInternalMessage, don't insert "if (isBounced) return" }; - int method_id = EMPTY_METHOD_ID; + int tvm_method_id = EMPTY_TVM_METHOD_ID; int flags; + FunctionInlineMode inline_mode; + int n_times_called = 0; // calculated while building call graph; 9999 for recursions + + std::string method_name; // for `fun Container.store` here is "store" + AnyTypeV receiver_type_node; // for `fun Container.store` here is `Container` + TypePtr receiver_type = nullptr; // = resolved receiver_type_node std::vector parameters; std::vector arg_order, ret_order; - TypePtr declared_return_type; // may be nullptr, meaning "auto infer" + AnyTypeV return_type_node; // may be nullptr, meaning "auto infer" + TypePtr declared_return_type = nullptr; // = resolved return_type_node TypePtr inferred_return_type = nullptr; // assigned on type inferring TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) const GenericsDeclaration* genericTs; - const GenericsInstantiation* instantiationTs; + const GenericsSubstitutions* substitutedTs; + FunctionPtr base_fun_ref = nullptr; // for `f`, here is `f` FunctionBody body; - AnyV ast_root; // V for user-defined (not builtin) + AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), loc) + , flags(initial_flags) + , inline_mode(inline_mode) + , method_name(std::move(method_name)) + , receiver_type_node(receiver_type_node) + , parameters(std::move(parameters)) + , return_type_node(return_type_node) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , body(body) + , ast_root(ast_root) { + } + FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , inline_mode(inline_mode) + , method_name(std::move(method_name)) + , receiver_type_node(nullptr) + , receiver_type(receiver_type) , parameters(std::move(parameters)) + , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) , genericTs(genericTs) - , instantiationTs(instantiationTs) + , substitutedTs(substitutedTs) , body(body) , ast_root(ast_root) { } @@ -141,38 +187,47 @@ struct FunctionData final : Symbol { int get_num_params() const { return static_cast(parameters.size()); } const LocalVarData& get_param(int idx) const { return parameters[idx]; } + LocalVarPtr find_param(std::string_view name) const; bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } + bool is_method() const { return !method_name.empty(); } + bool is_static_method() const { return is_method() && !does_accept_self(); } bool is_generic_function() const { return genericTs != nullptr; } - bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } + bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } - bool is_inline() const { return flags & flagInline; } - bool is_inline_ref() const { return flags & flagInlineRef; } + bool is_inlined_in_place() const { return inline_mode == FunctionInlineMode::inlineInPlace; } bool is_type_inferring_done() const { return flags & flagTypeInferringDone; } bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } bool is_implicit_return() const { return flags & flagImplicitReturn; } - bool is_get_method() const { return flags & flagGetMethod; } - bool is_method_id_not_empty() const { return method_id != EMPTY_METHOD_ID; } + bool is_contract_getter() const { return flags & flagContractGetter; } + bool has_tvm_method_id() const { return tvm_method_id != EMPTY_TVM_METHOD_ID; } bool is_entrypoint() const { return flags & flagIsEntrypoint; } bool has_mutate_params() const { return flags & flagHasMutateParams; } bool does_accept_self() const { return flags & flagAcceptsSelf; } bool does_return_self() const { return flags & flagReturnsSelf; } bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } + bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } + bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } + bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } + bool is_manual_on_bounce() const { return flags & flagManualOnBounce; } bool does_need_codegen() const; FunctionData* mutate() const { return const_cast(this); } + void assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr declared_return_type); void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); void assign_is_used_as_noncall(); void assign_is_implicit_return(); void assign_is_type_inferring_done(); void assign_is_really_used(); + void assign_inline_mode_in_place(); void assign_arg_order(std::vector&& arg_order); }; @@ -181,12 +236,13 @@ struct GlobalVarData final : Symbol { flagReallyUsed = 1, // calculated via dfs from used functions; unused globals are not codegenerated }; - TypePtr declared_type; // always exists, declaring globals without type is prohibited + AnyTypeV type_node; // `global a: int;` always exists, declaring globals without type is prohibited + TypePtr declared_type = nullptr; // = resolved type_node int flags = 0; - GlobalVarData(std::string name, SrcLocation loc, TypePtr declared_type) + GlobalVarData(std::string name, SrcLocation loc, AnyTypeV type_node) : Symbol(std::move(name), loc) - , declared_type(declared_type) { + , type_node(type_node) { } bool is_really_used() const { return flags & flagReallyUsed; } @@ -197,23 +253,127 @@ struct GlobalVarData final : Symbol { }; struct GlobalConstData final : Symbol { - ConstantValue value; - TypePtr declared_type; // may be nullptr + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr + TypePtr declared_type = nullptr; // = resolved type_node + TypePtr inferred_type = nullptr; + AnyExprV init_value; - GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, ConstantValue&& value) + GlobalConstData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV init_value) : Symbol(std::move(name), loc) - , value(std::move(value)) - , declared_type(declared_type) { + , type_node(type_node) + , init_value(init_value) { } - bool is_int_const() const { return value.is_int(); } - bool is_slice_const() const { return value.is_slice(); } + GlobalConstData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); + void assign_inferred_type(TypePtr inferred_type); + void assign_init_value(AnyExprV init_value); +}; - td::RefInt256 as_int_const() const { return value.as_int(); } - const std::string& as_slice_const() const { return value.as_slice(); } +struct AliasDefData final : Symbol { + AnyTypeV underlying_type_node; + TypePtr underlying_type = nullptr; // = resolved underlying_type_node - GlobalConstData* mutate() const { return const_cast(this); } + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + AliasDefPtr base_alias_ref = nullptr; // for `Response`, here is `Response` + AnyV ast_root; // V + + AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + : Symbol(std::move(name), loc) + , underlying_type_node(underlying_type_node) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { + } + + std::string as_human_readable() const; + + bool is_generic_alias() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_alias() const { return substitutedTs != nullptr; } + + AliasDefData* mutate() const { return const_cast(this); } + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); + void assign_resolved_type(TypePtr underlying_type); +}; + +struct StructFieldData final : Symbol { + int field_idx; + AnyTypeV type_node; + TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value; // nullptr if no default + + bool has_default_value() const { return default_value != nullptr; } + + StructFieldData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); + void assign_default_value(AnyExprV default_value); + + StructFieldData(std::string name, SrcLocation loc, int field_idx, AnyTypeV type_node, AnyExprV default_value) + : Symbol(std::move(name), loc) + , field_idx(field_idx) + , type_node(type_node) + , default_value(default_value) { + } +}; + +struct StructData final : Symbol { + enum class Overflow1023Policy { // annotation @overflow1023_policy above a struct + not_specified, + suppress, + }; + + struct PackOpcode { + int64_t pack_prefix; + int prefix_len; + + PackOpcode(int64_t pack_prefix, int prefix_len) + : pack_prefix(pack_prefix), prefix_len(prefix_len) {} + + bool exists() const { return prefix_len != 0; } + + std::string format_as_slice() const; // "x{...}" (or "b{...}") + }; + + std::vector fields; + PackOpcode opcode; + Overflow1023Policy overflow1023_policy; + + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + StructPtr base_struct_ref = nullptr; // for `Container`, here is `Container` + AnyV ast_root; // V + + int get_num_fields() const { return static_cast(fields.size()); } + StructFieldPtr get_field(int i) const { return fields.at(i); } + StructFieldPtr find_field(std::string_view field_name) const; + + bool is_generic_struct() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_struct() const { return substitutedTs != nullptr; } + + StructData* mutate() const { return const_cast(this); } + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); + + StructData(std::string name, SrcLocation loc, std::vector&& fields, PackOpcode opcode, Overflow1023Policy overflow1023_policy, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + : Symbol(std::move(name), loc) + , fields(std::move(fields)) + , opcode(opcode) + , overflow1023_policy(overflow1023_policy) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { + } + + std::string as_human_readable() const; +}; + +struct TypeReferenceUsedAsSymbol final : Symbol { + TypePtr resolved_type; + + TypeReferenceUsedAsSymbol(std::string name, SrcLocation loc, TypePtr resolved_type) + : Symbol(std::move(name), loc) + , resolved_type(resolved_type) { + } }; class GlobalSymbolTable { @@ -227,6 +387,10 @@ class GlobalSymbolTable { void add_function(FunctionPtr f_sym); void add_global_var(GlobalVarPtr g_sym); void add_global_const(GlobalConstPtr c_sym); + void add_type_alias(AliasDefPtr a_sym); + void add_struct(StructPtr s_sym); + + void replace_function(FunctionPtr f_sym); const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); @@ -235,5 +399,7 @@ class GlobalSymbolTable { }; const Symbol* lookup_global_symbol(std::string_view name); +FunctionPtr lookup_function(std::string_view name); +std::vector lookup_methods_with_name(std::string_view name); } // namespace tolk diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp index 7f939670b..17a2b4c84 100644 --- a/tolk/tolk-main.cpp +++ b/tolk/tolk-main.cpp @@ -50,6 +50,7 @@ void usage(const char* progname) { "-O\tSets optimization level (2 by default)\n" "-x\tEnables experimental options, comma-separated\n" "-S\tDon't include stack layout comments into Fift output\n" + "-L\tDon't include original lines from Tolk src into Fift output\n" "-e\tIncreases verbosity level (extra output into stderr)\n" "-v\tOutput version of Tolk and exit\n"; std::exit(2); @@ -148,19 +149,15 @@ static std::string auto_discover_stdlib_folder() { td::Result fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) { switch (kind) { case CompilerSettings::FsReadCallbackKind::Realpath: { - td::Result res_realpath; - if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) { - // import "@stdlib/filename" or import "@stdlib/filename.tolk" - std::string path = G.settings.stdlib_folder + static_cast(query + 7); - if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { - path += ".tolk"; - } - res_realpath = td::realpath(td::CSlice(path.c_str())); - } else { - // import "relative/to/cwd/path.tolk" - res_realpath = td::realpath(td::CSlice(query)); - } + bool is_stdlib = query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8); + std::string path = is_stdlib + ? G.settings.stdlib_folder + static_cast(query + 7) + : static_cast(query); + if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { + path += ".tolk"; + } + td::Result res_realpath = td::realpath(td::CSlice(path.c_str())); if (res_realpath.is_error()) { // note, that for non-existing files, `realpath()` on Linux/Mac returns an error, // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file" @@ -214,7 +211,7 @@ class StdCoutRedirectToFile { int main(int argc, char* const argv[]) { int i; - while ((i = getopt(argc, argv, "o:b:O:x:Sevh")) != -1) { + while ((i = getopt(argc, argv, "o:b:O:x:SLevh")) != -1) { switch (i) { case 'o': G.settings.output_filename = optarg; @@ -231,6 +228,9 @@ int main(int argc, char* const argv[]) { case 'S': G.settings.stack_layout_comments = false; break; + case 'L': + G.settings.tolk_src_as_line_comments = false; + break; case 'e': G.settings.verbosity++; break; diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index bbea63ffd..77d745eac 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.9.0"; +constexpr const char* TOLK_VERSION = "1.0.0"; } // namespace tolk diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp index e74589ce8..b5149481b 100644 --- a/tolk/tolk-wasm.cpp +++ b/tolk/tolk-wasm.cpp @@ -40,12 +40,14 @@ static td::Result compile_internal(char *config_json) { TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2)); TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false)); + TRY_RESULT(src_line_comments, td::get_json_object_bool_field(config, "withSrcLineComments", true, false)); TRY_RESULT(entrypoint_filename, td::get_json_object_string_field(config, "entrypointFileName", false)); TRY_RESULT(experimental_options, td::get_json_object_string_field(config, "experimentalOptions", true)); G.settings.verbosity = 0; G.settings.optimization_level = std::max(0, opt_level); G.settings.stack_layout_comments = stack_comments; + G.settings.tolk_src_as_line_comments = src_line_comments; if (!experimental_options.empty()) { G.settings.parse_experimental_options_cmd_arg(experimental_options.c_str()); } @@ -55,7 +57,7 @@ static td::Result compile_internal(char *config_json) { std::cerr.rdbuf(errs.rdbuf()); int exit_code = tolk_proceed(entrypoint_filename); if (exit_code != 0) { - return td::Status::Error("Tolk compilation error: " + errs.str()); + return td::Status::Error(errs.str()); } TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 71d1969de..69b03f20a 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -57,6 +57,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_register_global_symbols(); pipeline_resolve_identifiers_and_assign_symbols(); + pipeline_resolve_types_and_aliases(); pipeline_calculate_rvalue_lvalue(); pipeline_infer_types_and_calls_and_fields(); pipeline_check_inferred_types(); @@ -65,6 +66,10 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_pure_impure_operations(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); + pipeline_detect_inline_in_place(); + pipeline_check_serialized_fields(); + pipeline_lazy_load_insertions(); + pipeline_transform_onInternalMessage(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); @@ -77,7 +82,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { } catch (ParseError& error) { std::cerr << error << std::endl; return 2; - } catch (UnexpectedASTNodeType& error) { + } catch (UnexpectedASTNodeKind& error) { std::cerr << "fatal: " << error.what() << std::endl; std::cerr << "It's a compiler bug, please report to developers" << std::endl; return 2; diff --git a/tolk/tolk.h b/tolk/tolk.h index 3f00d0d4b..9bef3c4df 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -34,13 +34,18 @@ namespace tolk { GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN void on_assertion_failed(const char *description, const char *file_name, int line_number); +// fire a general error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + /* * * ABSTRACT CODE * */ -typedef int var_idx_t; typedef int const_idx_t; struct TmpVar { @@ -68,7 +73,6 @@ struct VarDescr { enum { _Last = 1, _Unused = 2 }; int flags; enum { - _Const = 16, _Int = 32, _Zero = 64, _NonZero = 128, @@ -79,16 +83,15 @@ struct VarDescr { _Even = 16384, _Odd = 32768, }; - static constexpr int ConstZero = _Const | _Int | _Zero | _Pos | _Neg | _Finite | _Even; - static constexpr int ConstOne = _Const | _Int | _NonZero | _Pos | _Finite | _Odd; - static constexpr int ConstTrue = _Const | _Int | _NonZero | _Neg | _Finite | _Odd; + static constexpr int ConstZero = _Int | _Zero | _Pos | _Neg | _Finite | _Even; + static constexpr int ConstOne = _Int | _NonZero | _Pos | _Finite | _Odd; + static constexpr int ConstTrue = _Int | _NonZero | _Neg | _Finite | _Odd; static constexpr int ValBit = _Int | _Pos | _Finite; static constexpr int ValBool = _Int | _Neg | _Finite; static constexpr int FiniteInt = _Int | _Finite; static constexpr int FiniteUInt = _Int | _Finite | _Pos; int val; td::RefInt256 int_const; - std::string str_const; explicit VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } @@ -120,7 +123,12 @@ struct VarDescr { return val & _Odd; } bool is_int_const() const { - return (val & (_Int | _Const)) == (_Int | _Const) && int_const.not_null(); +#ifdef TOLK_DEBUG + if (int_const.not_null()) { + tolk_assert(val & _Int); + } +#endif + return int_const.not_null(); } bool always_nonpos() const { return val & _Neg; @@ -151,7 +159,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); - void set_const(std::string value); + void set_const(const std::string& value); void operator+=(const VarDescr& y) { flags &= y.flags; } @@ -258,7 +266,6 @@ struct Stack; struct Op { enum OpKind { - _Undef, _Nop, _Call, _CallInd, @@ -279,53 +286,53 @@ struct Op { _SliceConst, }; OpKind cl; - enum { _Disabled = 1, _NoReturn = 4, _Impure = 24 }; + enum { _Disabled = 1, _NoReturn = 2, _Impure = 4, _ArgOrderAlreadyEqualsAsm = 8 }; int flags; std::unique_ptr next; FunctionPtr f_sym = nullptr; GlobalVarPtr g_sym = nullptr; - SrcLocation where; + SrcLocation loc; VarDescrList var_info; std::vector args; std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; std::string str_const; - Op(SrcLocation _where = {}, OpKind _cl = _Undef) : cl(_cl), flags(0), f_sym(nullptr), where(_where) { + Op(SrcLocation loc, OpKind cl) : cl(cl), flags(0), loc(loc) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left) { + Op(SrcLocation loc, OpKind cl, const std::vector& left) + : cl(cl), flags(0), loc(loc), left(left) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left) + : cl(cl), flags(0), loc(loc), left(std::move(left)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, td::RefInt256 _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), int_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, td::RefInt256 int_const) + : cl(cl), flags(0), loc(loc), left(left), int_const(std::move(int_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, std::string _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), str_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, std::string str_const) + : cl(cl), flags(0), loc(loc), left(left), str_const(std::move(str_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right) + : cl(cl), flags(0), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right) + : cl(cl), flags(0), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(_left), right(_right) { + : cl(cl), flags(0), f_sym(_fun), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, - FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, + FunctionPtr fun_ref) + : cl(cl), flags(0), f_sym(fun_ref), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, - GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, + GlobalVarPtr glob_ref) + : cl(cl), flags(0), g_sym(glob_ref), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(std::move(_left)), right(std::move(_right)) { + : cl(cl), flags(0), g_sym(_gvar), loc(loc), left(std::move(left)), right(std::move(right)) { } bool disabled() const { return flags & _Disabled; } @@ -339,6 +346,9 @@ struct Op { bool impure() const { return flags & _Impure; } void set_impure_flag(); + bool arg_order_already_equals_asm() const { return flags & _ArgOrderAlreadyEqualsAsm; } + void set_arg_order_already_equals_asm_flag(); + void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; @@ -351,6 +361,7 @@ struct Op { bool set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list); bool set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list); void prepare_args(VarDescrList values); + void maybe_swap_builtin_args_to_compile(); VarDescrList fwd_analyze(VarDescrList values); bool mark_noreturn(); bool is_empty() const { @@ -399,45 +410,46 @@ typedef std::vector StackLayout; typedef std::pair var_const_idx_t; typedef std::vector StackLayoutExt; constexpr const_idx_t not_const = -1; -using Const = td::RefInt256; struct AsmOp { - enum Type { a_none, a_xchg, a_push, a_pop, a_const, a_custom, a_magic }; - Type t{a_none}; + enum Type { a_nop, a_comment, a_xchg, a_push, a_pop, a_const, a_custom }; + Type t; + SrcLocation loc; int indent{0}; int a, b; bool gconst{false}; std::string op; struct SReg { int idx; - SReg(int _idx) : idx(_idx) { + explicit SReg(int _idx) : idx(_idx) { } + int calc_out_strlen() const; }; AsmOp() = default; - AsmOp(Type _t) : t(_t) { + AsmOp(Type t, SrcLocation loc) : t(t), loc(loc) { } - AsmOp(Type _t, std::string _op) : t(_t), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, std::string _op) : t(t), loc(loc), op(std::move(_op)) { } - AsmOp(Type _t, int _a) : t(_t), a(_a) { + AsmOp(Type t, SrcLocation loc, int a) : t(t), loc(loc), a(a) { } - AsmOp(Type _t, int _a, std::string _op) : t(_t), a(_a), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, std::string _op) : t(t), loc(loc), a(a), op(std::move(_op)) { } - AsmOp(Type _t, int _a, int _b) : t(_t), a(_a), b(_b) { + AsmOp(Type t, SrcLocation loc, int a, int b) : t(t), loc(loc), a(a), b(b) { } - AsmOp(Type _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, int b, std::string op) : t(t), loc(loc), a(a), b(b), op(std::move(op)) { compute_gconst(); } - void out(std::ostream& os) const; - void out_indent_nl(std::ostream& os, bool no_nl = false) const; + int out(std::ostream& os) const; + int out_indented(std::ostream& os, bool print_src_line_above) const; std::string to_string() const; void compute_gconst() { gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); } bool is_nop() const { - return t == a_none && op.empty(); + return t == a_nop; } bool is_comment() const { - return t == a_none && !op.empty(); + return t == a_comment; } bool is_custom() const { return t == a_custom; @@ -484,80 +496,80 @@ struct AsmOp { bool is_gconst() const { return !a && b == 1 && (t == a_const || gconst); } - static AsmOp Nop() { - return AsmOp(a_none); + static AsmOp Nop(SrcLocation loc) { + return AsmOp(a_nop, loc); } - static AsmOp Xchg(int a, int b = 0) { - return a == b ? AsmOp(a_none) : (a < b ? AsmOp(a_xchg, a, b) : AsmOp(a_xchg, b, a)); + static AsmOp Xchg(SrcLocation loc, int a, int b = 0) { + return a == b ? AsmOp(a_nop, loc) : (a < b ? AsmOp(a_xchg, loc, a, b) : AsmOp(a_xchg, loc, b, a)); } - static AsmOp Push(int a) { - return AsmOp(a_push, a); + static AsmOp Push(SrcLocation loc, int a) { + return AsmOp(a_push, loc, a); } - static AsmOp Pop(int a = 0) { - return AsmOp(a_pop, a); + static AsmOp Pop(SrcLocation loc, int a) { + return AsmOp(a_pop, loc, a); } - static AsmOp Xchg2(int a, int b) { - return make_stk2(a, b, "XCHG2", 0); + static AsmOp Xchg2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCHG2", 0); } - static AsmOp XcPu(int a, int b) { - return make_stk2(a, b, "XCPU", 1); + static AsmOp XcPu(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCPU", 1); } - static AsmOp PuXc(int a, int b) { - return make_stk2(a, b, "PUXC", 1); + static AsmOp PuXc(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUXC", 1); } - static AsmOp Push2(int a, int b) { - return make_stk2(a, b, "PUSH2", 2); + static AsmOp Push2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUSH2", 2); } - static AsmOp Xchg3(int a, int b, int c) { - return make_stk3(a, b, c, "XCHG3", 0); + static AsmOp Xchg3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCHG3", 0); } - static AsmOp Xc2Pu(int a, int b, int c) { - return make_stk3(a, b, c, "XC2PU", 1); + static AsmOp Xc2Pu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XC2PU", 1); } - static AsmOp XcPuXc(int a, int b, int c) { - return make_stk3(a, b, c, "XCPUXC", 1); + static AsmOp XcPuXc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPUXC", 1); } - static AsmOp XcPu2(int a, int b, int c) { - return make_stk3(a, b, c, "XCPU2", 3); + static AsmOp XcPu2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPU2", 3); } - static AsmOp PuXc2(int a, int b, int c) { - return make_stk3(a, b, c, "PUXC2", 3); + static AsmOp PuXc2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXC2", 3); } - static AsmOp PuXcPu(int a, int b, int c) { - return make_stk3(a, b, c, "PUXCPU", 3); + static AsmOp PuXcPu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXCPU", 3); } - static AsmOp Pu2Xc(int a, int b, int c) { - return make_stk3(a, b, c, "PU2XC", 3); + static AsmOp Pu2Xc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PU2XC", 3); } - static AsmOp Push3(int a, int b, int c) { - return make_stk3(a, b, c, "PUSH3", 3); + static AsmOp Push3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUSH3", 3); } - static AsmOp BlkSwap(int a, int b); - static AsmOp BlkPush(int a, int b); - static AsmOp BlkDrop(int a); - static AsmOp BlkDrop2(int a, int b); - static AsmOp BlkReverse(int a, int b); - static AsmOp make_stk2(int a, int b, const char* str, int delta); - static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); - static AsmOp IntConst(const td::RefInt256& x); - static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp BlkSwap(SrcLocation loc, int a, int b); + static AsmOp BlkPush(SrcLocation loc, int a, int b); + static AsmOp BlkDrop(SrcLocation loc, int a); + static AsmOp BlkDrop2(SrcLocation loc, int a, int b); + static AsmOp BlkReverse(SrcLocation loc, int a, int b); + static AsmOp make_stk2(SrcLocation loc, int a, int b, const char* str, int delta); + static AsmOp make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(SrcLocation loc, const td::RefInt256& x); + static AsmOp BoolConst(SrcLocation loc, bool f); + static AsmOp Const(SrcLocation loc, std::string push_op) { + return AsmOp(a_const, loc, 0, 1, std::move(push_op)); } - static AsmOp Const(int arg, const std::string& push_op); - static AsmOp Comment(const std::string& comment) { - return AsmOp(a_none, std::string{"// "} + comment); + static AsmOp Const(SrcLocation loc, int arg, const std::string& push_op); + static AsmOp Comment(SrcLocation loc, const std::string& comment) { + return AsmOp(a_comment, loc, std::string{"// "} + comment); } - static AsmOp Custom(const std::string& custom_op) { - return AsmOp(a_custom, 255, 255, custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op) { + return AsmOp(a_custom, loc, 255, 255, custom_op); } - static AsmOp Parse(const std::string& custom_op); - static AsmOp Custom(const std::string& custom_op, int args, int retv = 1) { - return AsmOp(a_custom, args, retv, custom_op); + static AsmOp Parse(SrcLocation loc, const std::string& custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, loc, args, retv, custom_op); } - static AsmOp Parse(std::string custom_op, int args, int retv = 1); - static AsmOp Tuple(int a); - static AsmOp UnTuple(int a); + static AsmOp Parse(SrcLocation loc, std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(SrcLocation loc, int a); + static AsmOp UnTuple(SrcLocation loc, int a); }; inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { @@ -572,38 +584,19 @@ struct AsmOpList { std::vector list_; int indent_{0}; const std::vector* var_names_{nullptr}; - std::vector constants_; + std::vector constants_; bool retalt_{false}; bool retalt_inserted_{false}; void out(std::ostream& os, int mode = 0) const; AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { } - template - AsmOpList& add(Args&&... args) { - append(AsmOp(std::forward(args)...)); + AsmOpList& operator<<(AsmOp&& op) { + list_.emplace_back(op); adjust_last(); return *this; } - bool append(const AsmOp& op) { - list_.push_back(op); - adjust_last(); - return true; - } - bool append(const std::vector& ops); - bool append(std::initializer_list ops) { - return append(std::vector(std::move(ops))); - } - AsmOpList& operator<<(const AsmOp& op) { - return add(op); - } - AsmOpList& operator<<(AsmOp&& op) { - return add(std::move(op)); - } - AsmOpList& operator<<(std::string str) { - return add(AsmOp::Type::a_custom, 255, 255, str); - } - const_idx_t register_const(Const new_const); - Const get_const(const_idx_t idx); + const_idx_t register_const(td::RefInt256 new_const); + td::RefInt256 get_const(const_idx_t idx); void show_var_ext(std::ostream& os, std::pair idx_pair) const; void adjust_last() { if (list_.back().is_nop()) { @@ -618,11 +611,8 @@ struct AsmOpList { void undent() { --indent_; } - void set_indent(int new_indent) { - indent_ = new_indent; - } - void insert(size_t pos, std::string str) { - insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + void insert(size_t pos, SrcLocation loc, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, loc, 255, 255, std::move(str))); } void insert(size_t pos, const AsmOp& op) { auto ip = list_.begin() + pos; @@ -636,11 +626,6 @@ struct AsmOpList { } }; -inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { - op_list.out(os); - return os; -} - struct AsmOpCons { std::unique_ptr car; std::unique_ptr cdr; @@ -722,9 +707,6 @@ struct StackTransform { bool almost_equal(const StackTransform& other) const { return equal(other, true); } - bool operator==(const StackTransform& other) const { - return dp == other.dp && almost_equal(other); - } bool operator<=(const StackTransform& other) const { return dp <= other.dp && almost_equal(other); } @@ -739,28 +721,6 @@ struct StackTransform { return get(i); } bool set(int i, int v, bool relaxed = false); - int operator()(int i) const { - return get(i); - } - class Pos { - StackTransform& t_; - int p_; - - public: - Pos(StackTransform& t, int p) : t_(t), p_(p) { - } - Pos& operator=(const Pos& other) = delete; - operator int() const { - return t_.get(p_); - } - const Pos& operator=(int v) const { - t_.set(p_, v); - return *this; - } - }; - Pos operator[](int i) { - return Pos(*this, i); - } static const StackTransform rot; static const StackTransform rot_rev; bool is_id() const { @@ -829,8 +789,6 @@ struct StackTransform { void show(std::ostream& os, int mode = 0) const; static StackTransform Xchg(int i, int j, bool relaxed = false); - static StackTransform Push(int i); - static StackTransform Pop(int i); private: int try_load(int& i, int offs = 0) const; // returns A[i++].first + offs or inf_x @@ -851,7 +809,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op); */ struct Optimizer { - static constexpr int optimize_depth = 20; + static constexpr int optimize_depth = 30; AsmOpConsList code_; int l_{0}, l2_{0}, p_, pb_, q_, indent_; bool debug_{false}; @@ -875,7 +833,6 @@ struct Optimizer { bool find(); bool optimize(); bool compute_stack_transforms(); - bool say(std::string str) const; bool show_stack_transforms() const; void show_head() const; void show_left() const; @@ -940,6 +897,17 @@ struct Optimizer { bool is_nip_seq(int* i, int* j); bool is_pop_blkdrop(int* i, int* k); bool is_2pop_blkdrop(int* i, int* j, int* k); + + bool detect_rewrite_big_THROW(); + bool detect_rewrite_MY_store_int(); + bool detect_rewrite_MY_skip_bits(); + bool detect_rewrite_NEWC_PUSH_STUR(); + bool detect_rewrite_LDxx_DROP(); + bool detect_rewrite_SWAP_symmetric(); + bool detect_rewrite_SWAP_PUSH_STUR(); + bool detect_rewrite_SWAP_STxxxR(); + bool detect_rewrite_NOT_THROWIF(); + AsmOpConsList extract_code(); }; @@ -951,17 +919,17 @@ struct Stack { StackLayoutExt s; AsmOpList& o; enum { - _StkCmt = 1, _CptStkCmt = 2, _DisableOut = 128, _Shown = 256, + _StackComments = 1, _LineComments = 2, _DisableOut = 128, _Shown = 256, _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; - Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { + Stack(AsmOpList& _o, int _mode) : o(_o), mode(_mode) { } - Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode = 0) : s(_s), o(_o), mode(_mode) { + Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode) : s(_s), o(_o), mode(_mode) { } - Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode = 0) : s(std::move(_s)), o(_o), mode(_mode) { + Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode) : s(std::move(_s)), o(_o), mode(_mode) { } int depth() const { return (int)s.size(); @@ -998,55 +966,56 @@ struct Stack { void forget_const(); void validate(int i) const { if (i > 255) { - throw Fatal{"Too deep stack"}; + throw Fatal("Too deep stack"); } tolk_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; } - void issue_pop(int i); - void issue_push(int i); - void issue_xchg(int i, int j); - int drop_vars_except(const VarDescrList& var_info, int excl_var = 0x80000000); + void issue_pop(SrcLocation loc, int i); + void issue_push(SrcLocation loc, int i); + void issue_xchg(SrcLocation loc, int i, int j); + int drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var = 0x80000000); void forget_var(var_idx_t idx); void push_new_var(var_idx_t idx); void push_new_const(var_idx_t idx, const_idx_t cidx); void assign_var(var_idx_t new_idx, var_idx_t old_idx); - void do_copy_var(var_idx_t new_idx, var_idx_t old_idx); - void enforce_state(const StackLayout& req_stack); - void rearrange_top(const StackLayout& top, std::vector last); - void rearrange_top(var_idx_t top, bool last); + void do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(SrcLocation loc, const StackLayout& req_stack); + void rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last); + void rearrange_top(SrcLocation loc, var_idx_t top, bool last); void merge_const(const Stack& req_stack); - void merge_state(const Stack& req_stack); + void merge_state(SrcLocation loc, const Stack& req_stack); void show(); void opt_show() { - if ((mode & (_StkCmt | _Shown)) == _StkCmt) { + if ((mode & (_StackComments | _Shown)) == _StackComments) { show(); } } bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers(int callxargs_count) { + void apply_wrappers(SrcLocation loc, int callxargs_count) { + int pos0 = (mode & _StackComments && !o.list_.empty() && o.list_[0].is_comment()) ? 1 : 0; bool is_inline = mode & _InlineFunc; if (o.retalt_inserted_) { - o.insert(0, "SAMEALTSAVE"); - o.insert(0, "c2 SAVE"); + o.insert(pos0, loc, "SAMEALTSAVE"); + o.insert(pos0, loc, "c2 SAVE"); } if (callxargs_count != -1 || (is_inline && o.retalt_)) { o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + o.insert(pos0, loc, "CONT:<{"); + o << AsmOp::Custom(loc, "}>"); if (callxargs_count != -1) { if (callxargs_count <= 15) { - o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " -1 CALLXARGS"); } else { tolk_assert(callxargs_count <= 254); - o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); } } else { - o << "EXECUTE"; + o << AsmOp::Custom(loc, "EXECUTE"); } } } @@ -1061,32 +1030,44 @@ struct Stack { typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; -inline simple_compile_func_t make_simple_compile(AsmOp op) { - return [op](std::vector& out, std::vector& in, SrcLocation) -> AsmOp { return op; }; -} - struct FunctionBodyBuiltin { simple_compile_func_t simple_compile; explicit FunctionBodyBuiltin(simple_compile_func_t compile) : simple_compile(std::move(compile)) {} - void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation where) const; + void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; }; struct FunctionBodyAsm { std::vector ops; void set_code(std::vector&& code); - void compile(AsmOpList& dest) const; + void compile(AsmOpList& dest, SrcLocation loc) const; +}; + +struct LazyVariableLoadedState; + +// LazyVarRefAtCodegen is a mutable state of a variable assigned by `lazy` operator: +// > var p = lazy Point.fromSlice(s) +// When inlining a method `p.getX()`, `self` also becomes lazy, pointing to the same state. +struct LazyVarRefAtCodegen { + LocalVarPtr var_ref; + const LazyVariableLoadedState* var_state; + + LazyVarRefAtCodegen(LocalVarPtr var_ref, const LazyVariableLoadedState* var_state) + : var_ref(var_ref), var_state(var_state) {} }; struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; std::string name; - SrcLocation loc; + SrcLocation forced_loc; std::vector vars; + std::vector lazy_variables; + std::vector* inline_rvect_out = nullptr; + bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; #ifdef TOLK_DEBUG @@ -1094,12 +1075,15 @@ struct CodeBlob { #endif std::stack*> cur_ops_stack; bool require_callxargs = false; - CodeBlob(std::string name, SrcLocation loc, FunctionPtr fun_ref) - : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(std::move(name)), loc(loc), cur_ops(&ops) { + explicit CodeBlob(FunctionPtr fun_ref) + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(fun_ref->name), cur_ops(&ops) { } template Op& emplace_back(Args&&... args) { Op& res = *(*cur_ops = std::make_unique(args...)); + if (forced_loc.is_defined()) { + res.loc = forced_loc; + } cur_ops = &(res.next); #ifdef TOLK_DEBUG _vector_of_ops.push_back(&res); @@ -1116,6 +1100,7 @@ struct CodeBlob { #endif return ir_idx; } + var_idx_t create_int(SrcLocation loc, int64_t value, const char* desc); bool compute_used_code_vars(); bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; void print(std::ostream& os, int flags = 0) const; @@ -1134,22 +1119,22 @@ struct CodeBlob { close_blk(location); pop_cur(); } + const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; + const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); - void generate_code(AsmOpList& out_list, int mode = 0); - void generate_code(std::ostream& os, int mode = 0, int indent = 0); + void generate_code(std::ostream& os, int mode = 0, int indent = 0) const; }; // defined in builtins.cpp -AsmOp exec_arg_op(std::string op, long long arg); -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv = 1); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv = 1); -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv = 1); -AsmOp push_const(td::RefInt256 x); +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(SrcLocation loc, td::RefInt256 x); void define_builtins(); +void patch_builtins_after_stdlib_loaded(); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d73625c20..cf89edb0b 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -15,26 +15,28 @@ along with TON Blockchain Library. If not, see . */ #include "type-system.h" -#include "lexer.h" #include "platform-utils.h" +#include "generics-helpers.h" #include "compiler-state.h" +#include #include namespace tolk { /* - * This class stores a big hashtable [hash => TypeData] + * This class stores a big hashtable [hash => TypePtr] * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. - * That's why all allocated TypeData objects are unique, storing unique type_id. + * That's why all allocated TypeData objects are unique, and can be compared as pointers. + * But compare pointers carefully due to type aliases. */ -class TypeDataTypeIdCalculation { +class TypeDataHasherForUnique { uint64_t cur_hash; int children_flags_mask = 0; static std::unordered_map all_unique_occurred_types; public: - explicit TypeDataTypeIdCalculation(uint64_t initial_arbitrary_unique_number) + explicit TypeDataHasherForUnique(uint64_t initial_arbitrary_unique_number) : cur_hash(initial_arbitrary_unique_number) {} void feed_hash(uint64_t val) { @@ -46,14 +48,10 @@ class TypeDataTypeIdCalculation { } void feed_child(TypePtr inner) { - feed_hash(inner->type_id); + feed_hash(reinterpret_cast(inner)); children_flags_mask |= inner->flags; } - uint64_t type_id() const { - return cur_hash; - } - int children_flags() const { return children_flags_mask; } @@ -67,14 +65,69 @@ class TypeDataTypeIdCalculation { GNU_ATTRIBUTE_NOINLINE TypePtr register_unique(TypePtr newly_created) const { #ifdef TOLK_DEBUG - assert(newly_created->type_id == cur_hash); + assert(all_unique_occurred_types.find(cur_hash) == all_unique_occurred_types.end()); #endif all_unique_occurred_types[cur_hash] = newly_created; return newly_created; } }; -std::unordered_map TypeDataTypeIdCalculation::all_unique_occurred_types; +/* + * This class stores a hashtable [TypePtr => type_id] + * We need type_id to support union types, that are stored as tagged unions on a stack. + * Every type that can be contained inside a union, has type_id. + * Some type_id are predefined (1 = int, etc.), but all user-defined types are assigned type_id. + */ +class TypeIdCalculation { + static int last_type_id; + static std::unordered_map map_ptr_to_type_id; + static std::vector instantiated_structs; + +public: + static int assign_type_id(TypePtr self) { + if (self->has_type_alias_inside()) { // type_id is calculated without aliases + self = unwrap_type_alias_deeply(self); // `(int,int)` equals `(IntAlias,IntAlias)`. + } + if (auto it = map_ptr_to_type_id.find(self); it != map_ptr_to_type_id.end()) { + return it->second; + } + + // make `Wrapper>` and `Wrapper>` and `Wrapper` have equal type_id + if (const TypeDataStruct* t_struct = self->try_as(); t_struct && t_struct->struct_ref->is_instantiation_of_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + instantiated_structs.push_back(struct_ref); + for (StructPtr another_ref : instantiated_structs) { + if (struct_ref != another_ref && self->equal_to(TypeDataStruct::create(another_ref))) { + auto it = map_ptr_to_type_id.find(TypeDataStruct::create(another_ref)); + assert(it != map_ptr_to_type_id.end()); + int type_id = it->second; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + } + } + + int type_id = ++last_type_id; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + + static TypePtr unwrap_type_alias_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataAlias* as_alias = child->try_as()) { + return as_alias->underlying_type->unwrap_alias(); + } + return child; + }); + } +}; + + +int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types +std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; +std::unordered_map TypeIdCalculation::map_ptr_to_type_id; +std::vector TypeIdCalculation::instantiated_structs; + TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; TypePtr TypeDataCell::singleton; @@ -82,7 +135,9 @@ TypePtr TypeDataSlice::singleton; TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; +TypePtr TypeDataAddress::singleton; TypePtr TypeDataNullLiteral::singleton; +TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; TypePtr TypeDataNever::singleton; TypePtr TypeDataVoid::singleton; @@ -95,36 +150,76 @@ void type_system_init() { TypeDataBuilder::singleton = new TypeDataBuilder; TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; + TypeDataAddress::singleton = new TypeDataAddress; TypeDataNullLiteral::singleton = new TypeDataNullLiteral; + TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; TypeDataNever::singleton = new TypeDataNever; TypeDataVoid::singleton = new TypeDataVoid; } +bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { + if (lhs->has_type_alias_inside()) { + lhs = TypeIdCalculation::unwrap_type_alias_deeply(lhs); + } + if (rhs->has_type_alias_inside()) { + rhs = TypeIdCalculation::unwrap_type_alias_deeply(rhs); + } + if (lhs == rhs) { + return true; + } + + if (const TypeDataUnion* lhs_union = lhs->try_as()) { + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); + } + } + if (const TypeDataStruct* lhs_struct = lhs->try_as(); lhs_struct && lhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + if (const TypeDataStruct* rhs_struct = rhs->try_as(); rhs_struct && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + return lhs_struct->struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref + && lhs_struct->struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); + } + } + return false; +} + +TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { + TypePtr unwrapped = lhs; + while (const TypeDataAlias* as_alias = unwrapped->try_as()) { + unwrapped = as_alias->underlying_type; + } + return unwrapped; +} + + // -------------------------------------------- // create() // // all constructors of TypeData classes are private, only TypeData*::create() is allowed -// each non-trivial create() method calculates hash (type_id) +// each non-trivial create() method calculates hash // and creates an object only if it isn't found in a global hashtable // -TypePtr TypeDataNullable::create(TypePtr inner) { - TypeDataTypeIdCalculation hash(1774084920039440885ULL); - hash.feed_child(inner); +TypePtr TypeDataAlias::create(AliasDefPtr alias_ref) { + TypeDataHasherForUnique hash(5694590762732189561ULL); + hash.feed_string(alias_ref->name); + hash.feed_child(alias_ref->underlying_type); if (TypePtr existing = hash.get_existing()) { return existing; } - // most types (int?, slice?, etc.), when nullable, still occupy 1 stack slot (holding TVM NULL at runtime) - // but for example for `(int, int)` we need an extra stack slot "null flag" - int width_on_stack = inner->can_hold_tvm_null_instead() ? 1 : inner->get_width_on_stack() + 1; - return hash.register_unique(new TypeDataNullable(hash.type_id(), hash.children_flags(), width_on_stack, inner)); + + TypePtr underlying_type = alias_ref->underlying_type; + if (underlying_type == TypeDataNullLiteral::create() || underlying_type == TypeDataNever::create() || underlying_type == TypeDataVoid::create()) { + return underlying_type; // aliasing these types is strange, don't store an alias + } + + return hash.register_unique(new TypeDataAlias(hash.children_flags(), alias_ref, underlying_type)); } TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { - TypeDataTypeIdCalculation hash(3184039965511020991ULL); + TypeDataHasherForUnique hash(3184039965511020991ULL); for (TypePtr param : params_types) { hash.feed_child(param); hash.feed_hash(767721); @@ -135,21 +230,47 @@ TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataFunCallable(hash.type_id(), hash.children_flags(), std::move(params_types), return_type)); + return hash.register_unique(new TypeDataFunCallable(hash.children_flags(), std::move(params_types), return_type)); } TypePtr TypeDataGenericT::create(std::string&& nameT) { - TypeDataTypeIdCalculation hash(9145033724911680012ULL); + TypeDataHasherForUnique hash(9145033724911680012ULL); hash.feed_string(nameT); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataGenericT(hash.type_id(), std::move(nameT))); + return hash.register_unique(new TypeDataGenericT(std::move(nameT))); +} + +TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { + TypeDataHasherForUnique hash(7451094818554079348ULL); // use it only to calculate children_flags + if (struct_ref) { + assert(alias_ref == nullptr && struct_ref->is_generic_struct()); + hash.feed_string(struct_ref->name); + } else { + assert(struct_ref == nullptr && alias_ref->is_generic_alias()); + hash.feed_string(alias_ref->name); + } + for (TypePtr argT : type_arguments) { + hash.feed_child(argT); + } + + return new TypeDataGenericTypeWithTs(hash.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); +} + +TypePtr TypeDataStruct::create(StructPtr struct_ref) { + TypeDataHasherForUnique hash(8315986401043319583ULL); + hash.feed_string(struct_ref->name); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataStruct(struct_ref)); } TypePtr TypeDataTensor::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(3159238551239480381ULL); + TypeDataHasherForUnique hash(3159238551239480381ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(819613); @@ -158,15 +279,11 @@ TypePtr TypeDataTensor::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; - for (TypePtr item : items) { - width_on_stack += item->get_width_on_stack(); - } - return hash.register_unique(new TypeDataTensor(hash.type_id(), hash.children_flags(), width_on_stack, std::move(items))); + return hash.register_unique(new TypeDataTensor(hash.children_flags(), std::move(items))); } -TypePtr TypeDataTypedTuple::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(9189266157349499320ULL); +TypePtr TypeDataBrackets::create(std::vector&& items) { + TypeDataHasherForUnique hash(9189266157349499320ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(735911); @@ -175,18 +292,230 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); + return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); +} + +TypePtr TypeDataIntN::create(int n_bits, bool is_unsigned, bool is_variadic) { + TypeDataHasherForUnique hash(1678330938771108027ULL); + hash.feed_hash(n_bits); + hash.feed_hash(is_unsigned); + hash.feed_hash(is_variadic); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataIntN(n_bits, is_unsigned, is_variadic)); +} + +TypePtr TypeDataBitsN::create(int n_width, bool is_bits) { + TypeDataHasherForUnique hash(7810988137199333041ULL); + hash.feed_hash(n_width); + hash.feed_hash(is_bits); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataBitsN(n_width, is_bits)); +} + +TypePtr TypeDataUnion::create(std::vector&& variants) { + TypeDataHasherForUnique hash(8719233194368471403ULL); + for (TypePtr variant : variants) { + hash.feed_child(variant); + hash.feed_hash(817663); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + + // for generics, like `f(v: T?)`, no type_id exists + // in this case, don't try to flatten: we have no info + // after instantiation, a new union type (with resolved variants) will be created + bool not_ready_yet = false; + for (TypePtr variant : variants) { + not_ready_yet |= variant->has_genericT_inside(); + } + if (not_ready_yet) { + TypePtr or_null = nullptr; + if (variants.size() == 2) { + if (variants[0] == TypeDataNullLiteral::create() || variants[1] == TypeDataNullLiteral::create()) { + or_null = variants[variants[0] == TypeDataNullLiteral::create()]; + } + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(variants))); + } + + // flatten variants and remove duplicates + // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants + std::vector flat_variants; + flat_variants.reserve(variants.size()); + for (TypePtr variant : variants) { + if (const TypeDataUnion* nested_union = variant->unwrap_alias()->try_as()) { + for (TypePtr nested_variant : nested_union->variants) { + append_union_type_variant(nested_variant, flat_variants); + } + } else { + append_union_type_variant(variant, flat_variants); + } + } + // detect, whether it's `T?` or `T1 | T2 | ...` + TypePtr or_null = nullptr; + if (flat_variants.size() == 2) { + if (flat_variants[0] == TypeDataNullLiteral::create() || flat_variants[1] == TypeDataNullLiteral::create()) { + or_null = flat_variants[flat_variants[0] == TypeDataNullLiteral::create()]; + } + } + + if (flat_variants.size() == 1) { // `int | int` + return flat_variants[0]; + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(flat_variants))); } -TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { - TypeDataTypeIdCalculation hash(3680147223540048162ULL); - hash.feed_string(text); - // hash.feed_hash(*reinterpret_cast(&loc)); +TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { + // calculate exactly the same hash as for `T | null` to create std::vector only if type seen the first time + TypeDataHasherForUnique hash(8719233194368471403ULL); + hash.feed_child(nullable); + hash.feed_hash(817663); + hash.feed_child(TypeDataNullLiteral::create()); + hash.feed_hash(817663); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataUnresolved(hash.type_id(), std::move(text), loc)); + return create({nullable, TypeDataNullLiteral::create()}); +} + + +// -------------------------------------------- +// get_width_on_stack() +// +// calculate, how many stack slots the type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 +// it's calculated dynamically (not saved at TypeData*::create) to overcome problems with +// - recursive struct mentions (to create TypeDataStruct without knowing width of children) +// - uninitialized generics (that don't make any sense upon being instantiated) +// + +int TypeDataAlias::get_width_on_stack() const { + return underlying_type->get_width_on_stack(); +} + +int TypeDataGenericT::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataGenericTypeWithTs::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataStruct::get_width_on_stack() const { + int width_on_stack = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + width_on_stack += field_ref->declared_type->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataTensor::get_width_on_stack() const { + int width_on_stack = 0; + for (TypePtr item : items) { + width_on_stack += item->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataUnion::get_width_on_stack() const { + if (or_null && or_null->can_hold_tvm_null_instead()) { + return 1; + } + + // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) + int max_child_width = 0; + for (TypePtr i : variants) { + if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) + max_child_width = std::max(max_child_width, i->get_width_on_stack()); + } + } + return max_child_width + 1; +} + +int TypeDataNever::get_width_on_stack() const { + return 0; +} + +int TypeDataVoid::get_width_on_stack() const { + return 0; +} + + +// -------------------------------------------- +// get_type_id() +// +// in order to support union types, every type that can be stored inside a union has a unique type_id +// some are predefined (1 = int, etc. in .h file), the others are here +// + +int TypeDataAlias::get_type_id() const { + assert(!alias_ref->is_generic_alias()); + return underlying_type->get_type_id(); +} + +int TypeDataFunCallable::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataGenericT::get_type_id() const { + assert(false); // generics must have been instantiated in advance + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataGenericTypeWithTs::get_type_id() const { + assert(false); // `Wrapper` has to be resolved in advance + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataStruct::get_type_id() const { + assert(!struct_ref->is_generic_struct()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataTensor::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataBrackets::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataIntN::get_type_id() const { + switch (n_bits) { + case 8: return 42 + is_unsigned; // for common intN, use predefined small numbers + case 16: return 44 + is_unsigned; + case 32: return 46 + is_unsigned; + case 64: return 48 + is_unsigned; + case 128: return 50 + is_unsigned; + case 256: return 52 + is_unsigned; + default: return TypeIdCalculation::assign_type_id(this); + } +} + +int TypeDataBitsN::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataUnion::get_type_id() const { + assert(false); // a union can not be inside a union + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataUnknown::get_type_id() const { + assert(false); // unknown can not be inside a union + throw Fatal("unexpected get_type_id() call"); } @@ -197,10 +526,8 @@ TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { // only non-trivial implementations are here; trivial are defined in .h file // -std::string TypeDataNullable::as_human_readable() const { - std::string nested = inner->as_human_readable(); - bool embrace = inner->try_as(); - return embrace ? "(" + nested + ")?" : nested + "?"; +std::string TypeDataAlias::as_human_readable() const { + return alias_ref->name; } std::string TypeDataFunCallable::as_human_readable() const { @@ -216,6 +543,23 @@ std::string TypeDataFunCallable::as_human_readable() const { return result; } +std::string TypeDataGenericTypeWithTs::as_human_readable() const { + std::string result = struct_ref ? struct_ref->name : alias_ref->name; + result += '<'; + for (TypePtr argT : type_arguments) { + if (result[result.size() - 1] != '<') { + result += ", "; + } + result += argT->as_human_readable(); + } + result += '>'; + return result; +} + +std::string TypeDataStruct::as_human_readable() const { + return struct_ref->name; +} + std::string TypeDataTensor::as_human_readable() const { std::string result = "("; for (TypePtr item : items) { @@ -228,7 +572,7 @@ std::string TypeDataTensor::as_human_readable() const { return result; } -std::string TypeDataTypedTuple::as_human_readable() const { +std::string TypeDataBrackets::as_human_readable() const { std::string result = "["; for (TypePtr item : items) { if (result.size() > 1) { @@ -240,39 +584,40 @@ std::string TypeDataTypedTuple::as_human_readable() const { return result; } - -// -------------------------------------------- -// traverse() -// -// invokes a callback for TypeData itself and all its children -// only non-trivial implementations are here; by default (no children), `callback(this)` is executed -// - -void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { - callback(this); - inner->traverse(callback); +std::string TypeDataIntN::as_human_readable() const { + std::string s_int = is_variadic + ? is_unsigned ? "varuint" : "varint" + : is_unsigned ? "uint" : "int"; + return s_int + std::to_string(n_bits); } -void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr param : params_types) { - param->traverse(callback); - } - return_type->traverse(callback); +std::string TypeDataBitsN::as_human_readable() const { + std::string s_bits = is_bits ? "bits" : "bytes"; + return s_bits + std::to_string(n_width); } -void TypeDataTensor::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); +std::string TypeDataUnion::as_human_readable() const { + // stringify `T?`, not `T | null` + if (or_null) { + bool embrace = or_null->try_as(); + return embrace ? "(" + or_null->as_human_readable() + ")?" : or_null->as_human_readable() + "?"; } -} -void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); + std::string result; + for (TypePtr variant : variants) { + if (!result.empty()) { + result += " | "; + } + bool embrace = variant->try_as(); + if (embrace) { + result += "("; + } + result += variant->as_human_readable(); + if (embrace) { + result += ")"; + } } + return result; } @@ -284,10 +629,6 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { // only non-trivial implementations are here; by default (no children), `return callback(this)` is executed // -TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { - return callback(create(inner->replace_children_custom(callback))); -} - TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(params_types.size()); @@ -297,6 +638,15 @@ TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& ca return callback(create(std::move(mapped), return_type->replace_children_custom(callback))); } +TypePtr TypeDataGenericTypeWithTs::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(type_arguments.size()); + for (TypePtr argT : type_arguments) { + mapped.push_back(argT->replace_children_custom(callback)); + } + return callback(create(struct_ref, alias_ref, std::move(mapped))); +} + TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); @@ -306,7 +656,7 @@ TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callbac return callback(create(std::move(mapped))); } -TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& callback) const { +TypePtr TypeDataBrackets::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); for (TypePtr item : items) { @@ -315,18 +665,45 @@ TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& cal return callback(create(std::move(mapped))); } +TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(variants.size()); + for (TypePtr variant : variants) { + mapped.push_back(variant->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + // -------------------------------------------- // can_rhs_be_assigned() // // on `var lhs: = rhs`, having inferred rhs_type, check that it can be assigned without any casts // the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) +// note, that `int8 | int16` is not assignable to `int` (even though both are assignable), +// because the only way to work with union types is to use `match`/`is` operators // +bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return underlying_type->can_rhs_be_assigned(rhs); +} + bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (rhs->try_as()) { + return true; + } + if (rhs == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -334,6 +711,9 @@ bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -341,6 +721,14 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { + if (rhs_struct->struct_ref->is_instantiation_of_generic_struct() && rhs_struct->struct_ref->base_struct_ref->name == "Cell") { + return true; // Cell to cell, e.g. `contract.setData(obj.toCell())` + } + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -348,13 +736,19 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); // note, that bitsN/address is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -362,6 +756,9 @@ bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -369,28 +766,28 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } -bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { +bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); // note, that slice is NOT automatically cast to address without `as` operator } -bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { +bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - if (rhs == TypeDataNullLiteral::create()) { - return true; - } - if (const TypeDataNullable* rhs_nullable = rhs->try_as()) { - return inner->can_rhs_be_assigned(rhs_nullable->inner); - } - if (inner->can_rhs_be_assigned(rhs)) { - return true; + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); } return rhs == TypeDataNever::create(); } @@ -399,14 +796,48 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataFunCallable* rhs_callable = rhs->try_as()) { + if (rhs_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!rhs_callable->params_types[i]->can_rhs_be_assigned(params_types[i])) { + return false; + } + if (!params_types[i]->can_rhs_be_assigned(rhs_callable->params_types[i])) { + return false; + } + } + return return_type->can_rhs_be_assigned(rhs_callable->return_type) && + rhs_callable->return_type->can_rhs_be_assigned(return_type); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); return false; } +bool TypeDataGenericTypeWithTs::can_rhs_be_assigned(TypePtr rhs) const { + return false; +} + +bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C + return equal_to(rhs_struct); + } + return rhs == TypeDataNever::create(); +} + bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -416,11 +847,14 @@ bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } -bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { - if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { +bool TypeDataBrackets::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_rhs_be_assigned(as_tuple->items[i])) { return false; @@ -428,22 +862,75 @@ bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } -bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { - return true; +bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } -bool TypeDataUnresolved::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); +bool TypeDataBitsN::can_rhs_be_assigned(TypePtr rhs) const { + // `slice` is NOT assignable to bitsN without `as` + // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return false; } -bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { +bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (calculate_exact_variant_to_fit_rhs(rhs)) { // `int` to `int | slice`, `int?` to `int8?`, `(int, null)` to `(int, T?) | slice` + return true; + } + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return has_all_variants_of(rhs_union); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + + return rhs == TypeDataNever::create(); +} + +bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } +bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { + return rhs == TypeDataNever::create(); +} + bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -459,77 +946,187 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { // note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` // +// common helper for union types: +// - `int as int?` is ok +// - `int8 as int16?` is ok (primitive 1-slot nullable don't store UTag, rules are less strict) +// - `int as int | int16` is ok (exact match one of types) +// - `int as slice | null` is NOT ok (no rhs subtype fits) +// - `int as int8 | int16` is NOT ok (ambiguity) +static bool can_be_casted_to_union(TypePtr self, const TypeDataUnion* rhs_union) { + if (rhs_union->is_primitive_nullable()) { // casting to primitive 1-slot nullable + return self == TypeDataNullLiteral::create() || self->can_be_casted_with_as_operator(rhs_union->or_null); + } + + return rhs_union->calculate_exact_variant_to_fit_rhs(self) != nullptr; +} + +bool TypeDataAlias::can_be_casted_with_as_operator(TypePtr cast_to) const { + return underlying_type->can_be_casted_with_as_operator(cast_to); +} + bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int` as `int?` / `int` as `int | slice` + return can_be_casted_to_union(this, to_union); + } + if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` + return true; + } + if (cast_to == TypeDataCoins::create()) { // `int` as `coins` + return true; + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (cast_to == TypeDataInt::create()) { + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const auto* to_intN = cast_to->try_as()) { + return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == this || cast_to == TypeDataInt::create(); + return cast_to == this; } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // cell as Cell + return to_struct->struct_ref->is_instantiation_of_generic_struct() && to_struct->struct_ref->base_struct_ref->name == "Cell"; + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` + return true; + } + if (cast_to == TypeDataAddress::create()) { + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return cast_to == this; } -bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { - return cast_to == this || cast_to->try_as(); +bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataSlice::create()) { + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; } -bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return inner->can_be_casted_with_as_operator(to_nullable->inner); +bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `null` to `T?` / `null` to `... | null` + return can_be_casted_to_union(this, to_union); } - return false; + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; } bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } - return this == cast_to; + if (const TypeDataFunCallable* to_callable = cast_to->try_as()) { + if (to_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!params_types[i]->can_be_casted_with_as_operator(to_callable->params_types[i])) { + return false; + } + if (!to_callable->params_types[i]->can_be_casted_with_as_operator(params_types[i])) { + return false; + } + } + return return_type->can_be_casted_with_as_operator(to_callable->return_type) && + to_callable->return_type->can_be_casted_with_as_operator(return_type); + } + return false; } bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } +bool TypeDataGenericTypeWithTs::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + +bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (cast_to == TypeDataCell::create()) { // Cell as cell + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // C> as C + return equal_to(to_struct); + } + return cast_to == this; +} + bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -539,14 +1136,20 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return false; } -bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { +bool TypeDataBrackets::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `[int, int]` as `tuple` + return true; + } + if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { return false; @@ -554,21 +1157,75 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); } return false; } -bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { - // 'unknown' can be cast to any TVM value - return cast_to->get_width_on_stack() == 1; +bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `int8` as `int32`, `int256` as `uint5`, anything + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8` as `int32?` + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } -bool TypeDataUnresolved::can_be_casted_with_as_operator(TypePtr cast_to) const { +bool TypeDataBitsN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == TypeDataSlice::create() || cast_to == TypeDataAddress::create(); +} + +bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `coins` as `int8` + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + if (cast_to == TypeDataInt::create()) { + return true; + } + return cast_to == this; +} + +bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8 | int16` as `int16 | int8 | slice` + if (to_union->is_primitive_nullable()) { + return or_null && or_null->can_be_casted_with_as_operator(to_union->or_null); + } + return to_union->has_all_variants_of(this); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } +bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { + // 'unknown' can be cast to any TVM value + return cast_to->get_width_on_stack() == 1; +} + bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } @@ -587,11 +1244,20 @@ bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { // though still, tricky situations like `(int, ())?` can still "embed" TVM NULL in parallel with original value // -bool TypeDataNullable::can_hold_tvm_null_instead() const { - if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead - return false; // only `int?` / `cell?` / `StructWith1IntField?` can - } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` - return !inner->can_hold_tvm_null_instead(); +bool TypeDataAlias::can_hold_tvm_null_instead() const { + return underlying_type->can_hold_tvm_null_instead(); +} + +bool TypeDataStruct::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // example that can hold null: `{ field: int }` + return false; // another example: `{ e: Empty, field: ((), int) }` + } // examples can NOT: `{ field1: int, field2: int }`, `{ field1: int? }` + for (StructFieldPtr field : struct_ref->fields) { + if (field->declared_type->get_width_on_stack() == 1 && !field->declared_type->can_hold_tvm_null_instead()) { + return false; + } + } + return true; } bool TypeDataTensor::can_hold_tvm_null_instead() const { @@ -606,6 +1272,13 @@ bool TypeDataTensor::can_hold_tvm_null_instead() const { return true; } +bool TypeDataUnion::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead + return false; // only `int?` / `cell?` / `StructWith1IntField?` can + } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` + return or_null && !or_null->can_hold_tvm_null_instead(); +} + bool TypeDataNever::can_hold_tvm_null_instead() const { return false; } @@ -615,139 +1288,76 @@ bool TypeDataVoid::can_hold_tvm_null_instead() const { } -// -------------------------------------------- -// parsing type from tokens -// -// here we implement parsing types (mostly after colon) to TypeData -// example: `var v: int` is TypeDataInt -// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataNullable(TypeDataBuilder), TypeDataTypedTuple(TypeDataCell)) -// example: `fun f(): ()` is TypeDataTensor() (an empty one) -// -// note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, -// and later, when all files are parsed and all symbols registered, such identifiers are resolved -// example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT -// see finalize_type_data() -// -// note, that `self` does not name a type, it can appear only as a return value of a function (parsed specially) -// when `self` appears as a type, it's parsed as TypeDataUnresolved, and later an error is emitted -// - -static TypePtr parse_type_expression(Lexer& lex); - -std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { - lex.expect(tok_op, s_op); - std::vector sub_types; - while (true) { - if (lex.tok() == tok_cl) { // empty lists allowed - lex.next(); - break; - } - - sub_types.emplace_back(parse_type_expression(lex)); - if (lex.tok() == tok_comma) { - lex.next(); - } else if (lex.tok() != tok_cl) { - lex.unexpected(s_cl); - } - } - return sub_types; -} - -std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { - return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); -} - -static TypePtr parse_simple_type(Lexer& lex) { - switch (lex.tok()) { - case tok_self: - case tok_identifier: { - SrcLocation loc = lex.cur_location(); - std::string_view str = lex.cur_str(); - lex.next(); - switch (str.size()) { - case 3: - if (str == "int") return TypeDataInt::create(); - break; - case 4: - if (str == "cell") return TypeDataCell::create(); - if (str == "void") return TypeDataVoid::create(); - if (str == "bool") return TypeDataBool::create(); - break; - case 5: - if (str == "slice") return TypeDataSlice::create(); - if (str == "tuple") return TypeDataTuple::create(); - if (str == "never") return TypeDataNever::create(); - break; - case 7: - if (str == "builder") return TypeDataBuilder::create(); - break; - case 12: - if (str == "continuation") return TypeDataContinuation::create(); - break; - default: - break; - } - return TypeDataUnresolved::create(std::string(str), loc); - } - case tok_null: - lex.next(); - return TypeDataNullLiteral::create(); - case tok_oppar: { - std::vector items = parse_nested_type_list_in_parenthesis(lex); - if (items.size() == 1) { - return items.front(); - } - return TypeDataTensor::create(std::move(items)); +// union types creation is a bit tricky: nested unions are flattened, duplicates are removed +// so, a resolved union type has variants, each with unique type_id +// (type_id is calculated with aliases erasure) +void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vector& out_unique_variants) { + for (TypePtr existing : out_unique_variants) { + if (existing->get_type_id() == variant->get_type_id()) { + return; } - case tok_opbracket: { - std::vector items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`"); - return TypeDataTypedTuple::create(std::move(items)); - } - default: - lex.unexpected(""); } -} -static TypePtr parse_type_nullable(Lexer& lex) { - TypePtr result = parse_simple_type(lex); + out_unique_variants.push_back(variant); +} - if (lex.tok() == tok_question) { - lex.next(); - result = TypeDataNullable::create(result); +bool TypeDataUnion::has_variant_with_type_id(int type_id) const { + for (TypePtr self_variant : variants) { + if (self_variant->get_type_id() == type_id) { + return true; + } } - - return result; + return false; } -static TypePtr parse_type_expression(Lexer& lex) { - TypePtr result = parse_type_nullable(lex); +bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { + for (TypePtr rhs_variant : rhs_type->variants) { + if (!has_variant_with_type_id(rhs_variant->get_type_id())) { + return false; + } + } + return true; +} - if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void` - lex.next(); - TypePtr return_type = parse_type_expression(lex); - std::vector params_types = {result}; - if (const auto* as_tensor = result->try_as()) { - params_types = as_tensor->items; +int TypeDataUnion::get_variant_idx(TypePtr lookup_variant) const { + for (int i = 0; i < size(); ++i) { + if (variants[i]->equal_to(lookup_variant)) { + return i; } - return TypeDataFunCallable::create(std::move(params_types), return_type); } + return -1; +} - if (lex.tok() != tok_bitwise_or) { - return result; +// given this = `T1 | T2 | ...` and rhs_type, find the only (not ambiguous) T_i that can accept it +TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const { + // primitive 1-slot nullable don't store type_id, they can be assigned less strict, like `int?` to `int16?` + if (const TypeDataUnion* rhs_union = rhs_type->unwrap_alias()->try_as()) { + if (is_primitive_nullable() && rhs_union->is_primitive_nullable() && or_null->can_rhs_be_assigned(rhs_union->or_null)) { + return this; + } + return nullptr; + } + // `int` to `int | int8` is okay: exact type matching + for (TypePtr variant : variants) { + if (variant->get_type_id() == rhs_type->get_type_id()) { + return variant; + } } - lex.error("union types are not supported yet"); + // find the only T_i; it would also be used for transition at IR generation, like `(int,null)` to `(int, User?) | int` + TypePtr first_covering = nullptr; + for (TypePtr variant : variants) { + if (variant->can_rhs_be_assigned(rhs_type)) { + if (first_covering) { + return nullptr; + } + first_covering = variant; + } + } + return first_covering; } -TypePtr parse_type_from_tokens(Lexer& lex) { - return parse_type_expression(lex); -} -// for internal usage only -TypePtr parse_type_from_string(std::string_view text) { - Lexer lex(text); - return parse_type_expression(lex); -} std::ostream& operator<<(std::ostream& os, TypePtr type_data) { return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); diff --git a/tolk/type-system.h b/tolk/type-system.h index 4b671e30d..e9a63c213 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -16,10 +16,10 @@ */ #pragma once -#include "src-file.h" -#include -#include +#include "fwd-declarations.h" #include +#include +#include namespace tolk { @@ -27,47 +27,37 @@ namespace tolk { * TypeData is both a user-given and an inferred type representation. * `int`, `cell`, `T`, `(int, [tuple])` are instances of TypeData. * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) - * returns one and the same pointer always. This "uniqueness" is called type_id, calculated before creation. + * returns one and the same pointer always. * - * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. - * See parse_type_from_tokens(). - * So, AST nodes which can have declared types (local/global variables and others) store a pointer to TypeData. - * - * Type inferring also creates TypeData for inferred expressions. All AST expression nodes have inferred_type. + * Type inferring creates TypeData for inferred expressions. All AST expression nodes have inferred_type. * For example, `1 + 2`, both operands are TypeDataInt, its result is also TypeDataInt. * Type checking also uses TypeData. For example, `var i: slice = 1 + 2`, at first rhs (TypeDataInt) is inferred, * then lhs (TypeDataSlice from declaration) is checked whether rhs can be assigned. * See can_rhs_be_assigned(). * - * Note, that while initial parsing Tolk files to AST, known types (`int`, `cell`, etc.) are created as-is, - * but user-defined types (`T`, `MyStruct`, `MyAlias`) are saved as TypeDataUnresolved. - * After all symbols have been registered, resolving identifiers step is executed, where particularly - * all TypeDataUnresolved instances are converted to a resolved type. At inferring, no unresolved remain. - * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. + * At the moment of parsing, types after colon `var v: (int, T)` are parsed to AST (AnyTypeV), + * and all symbols have been registered, AST representation resolved to TypeData, see pipe-resolve-types.cpp. */ class TypeData { - // all unique types have unique type_id; it's used both for allocating memory once and for tagged unions - const uint64_t type_id; // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; - // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 - const int width_on_stack; - friend class TypeDataTypeIdCalculation; + friend class TypeDataHasherForUnique; protected: enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, - flag_contains_unresolved_inside = 1 << 3, + flag_contains_type_alias_inside = 1 << 3, }; - explicit TypeData(uint64_t type_id, int flags_with_children, int width_on_stack) - : type_id(type_id) - , flags(flags_with_children) - , width_on_stack(width_on_stack) { + explicit TypeData(int flags_with_children) + : flags(flags_with_children) { } + static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); + static TypePtr unwrap_alias_slow_path(TypePtr lhs); + public: virtual ~TypeData() = default; @@ -76,16 +66,25 @@ class TypeData { return dynamic_cast(this); } - uint64_t get_type_id() const { return type_id; } - int get_width_on_stack() const { return width_on_stack; } + // how many slots on a stack this type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 + virtual int get_width_on_stack() const { + return 1; // most types occupy 1 stack slot (int, cell, slice, etc.) + } + + bool equal_to(TypePtr rhs) const { + return this == rhs || equal_to_slow_path(this, rhs); + } + TypePtr unwrap_alias() const { + return has_type_alias_inside() ? unwrap_alias_slow_path(this) : this; + } bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } - bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } + bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } - using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; + virtual int get_type_id() const = 0; virtual std::string as_human_readable() const = 0; virtual bool can_rhs_be_assigned(TypePtr rhs) const = 0; virtual bool can_be_casted_with_as_operator(TypePtr cast_to) const = 0; @@ -94,20 +93,44 @@ class TypeData { return true; } - virtual void traverse(const TraverserCallbackT& callback) const { - callback(this); - } - virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { return callback(this); } }; +/* + * `type AliasName = underlying_type` is an alias, which is fully interchangeable with its original type. + * It never occurs at runtime: at IR generation it's erased, replaced by an underlying type. + * But until IR generation, aliases exists, and `var t: MyTensor2 = (1,2)` is alias "MyTensor", not tensor (int,int). + * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. + * Note, that generic aliases, when instantiated, are inserted into in symtable (like structs and functions), + * so for `WrapperAlias` alias_ref points to a generic alias, and for `WrapperAlias` to an instantiated one. + */ +class TypeDataAlias final : public TypeData { + explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) + : TypeData(children_flags | flag_contains_type_alias_inside) + , alias_ref(alias_ref) + , underlying_type(underlying_type) {} + +public: + const AliasDefPtr alias_ref; + const TypePtr underlying_type; + + static TypePtr create(AliasDefPtr alias_ref); + + int get_width_on_stack() const override; + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `int` is TypeDataInt, representation of TVM int. */ class TypeDataInt final : public TypeData { - TypeDataInt() : TypeData(1ULL, 0, 1) {} + TypeDataInt() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -115,6 +138,7 @@ class TypeDataInt final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 1; } std::string as_human_readable() const override { return "int"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -125,7 +149,7 @@ class TypeDataInt final : public TypeData { * From the type system point of view, int and bool are different, not-autocastable types. */ class TypeDataBool final : public TypeData { - TypeDataBool() : TypeData(2ULL, 0, 1) {} + TypeDataBool() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -133,6 +157,7 @@ class TypeDataBool final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 2; } std::string as_human_readable() const override { return "bool"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -142,7 +167,7 @@ class TypeDataBool final : public TypeData { * `cell` is TypeDataCell, representation of TVM cell. */ class TypeDataCell final : public TypeData { - TypeDataCell() : TypeData(3ULL, 0, 1) {} + TypeDataCell() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -150,6 +175,7 @@ class TypeDataCell final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 3; } std::string as_human_readable() const override { return "cell"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -159,7 +185,7 @@ class TypeDataCell final : public TypeData { * `slice` is TypeDataSlice, representation of TVM slice. */ class TypeDataSlice final : public TypeData { - TypeDataSlice() : TypeData(4ULL, 0, 1) {} + TypeDataSlice() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -167,6 +193,7 @@ class TypeDataSlice final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 4; } std::string as_human_readable() const override { return "slice"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -176,7 +203,7 @@ class TypeDataSlice final : public TypeData { * `builder` is TypeDataBuilder, representation of TVM builder. */ class TypeDataBuilder final : public TypeData { - TypeDataBuilder() : TypeData(5ULL, 0, 1) {} + TypeDataBuilder() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -184,6 +211,7 @@ class TypeDataBuilder final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 5; } std::string as_human_readable() const override { return "builder"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -195,7 +223,7 @@ class TypeDataBuilder final : public TypeData { * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). */ class TypeDataTuple final : public TypeData { - TypeDataTuple() : TypeData(6ULL, 0, 1) {} + TypeDataTuple() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -203,6 +231,7 @@ class TypeDataTuple final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 6; } std::string as_human_readable() const override { return "tuple"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -213,7 +242,7 @@ class TypeDataTuple final : public TypeData { * It's like "untyped callable", not compatible with other types. */ class TypeDataContinuation final : public TypeData { - TypeDataContinuation() : TypeData(7ULL, 0, 1) {} + TypeDataContinuation() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -221,19 +250,18 @@ class TypeDataContinuation final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 7; } std::string as_human_readable() const override { return "continuation"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; /* - * `null` has TypeDataNullLiteral type. - * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. - * Note, that `var i = null`, though valid (i would be constant null), fires an "always-null" compilation error - * (it's much better for user to see an error here than when he passes this variable somewhere). + * `address` is TypeDataAddress — TVM slice under the hood, but since it's a very common use case, + * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). */ -class TypeDataNullLiteral final : public TypeData { - TypeDataNullLiteral() : TypeData(8ULL, 0, 1) {} +class TypeDataAddress final : public TypeData { + TypeDataAddress() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -241,33 +269,31 @@ class TypeDataNullLiteral final : public TypeData { public: static TypePtr create() { return singleton; } - std::string as_human_readable() const override { return "null"; } + int get_type_id() const override { return 8; } + std::string as_human_readable() const override { return "address"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; /* - * `T?` is "nullable T". - * It can be converted to T either with ! (non-null assertion operator) or with smart casts. + * `null` has TypeDataNullLiteral type. + * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. + * Note, that `var i = null`, though valid (i would be constant null), fires an "always-null" compilation error + * (it's much better for user to see an error here than when he passes this variable somewhere). */ -class TypeDataNullable final : public TypeData { - TypeDataNullable(uint64_t type_id, int children_flags, int width_on_stack, TypePtr inner) - : TypeData(type_id, children_flags, width_on_stack) - , inner(inner) {} - -public: - const TypePtr inner; +class TypeDataNullLiteral final : public TypeData { + TypeDataNullLiteral() : TypeData(0) {} - static TypePtr create(TypePtr inner); + static TypePtr singleton; + friend void type_system_init(); - bool is_primitive_nullable() const { return get_width_on_stack() == 1 && inner->get_width_on_stack() == 1; } +public: + static TypePtr create() { return singleton; } - std::string as_human_readable() const override; + int get_type_id() const override { return 0; } + std::string as_human_readable() const override { return "null"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; - TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; - bool can_hold_tvm_null_instead() const override; }; /* @@ -276,8 +302,8 @@ class TypeDataNullable final : public TypeData { * So, when assigning it to a variable `var cb = f`, this variable also has this type. */ class TypeDataFunCallable final : public TypeData { - TypeDataFunCallable(uint64_t type_id, int children_flags, std::vector&& params_types, TypePtr return_type) - : TypeData(type_id, children_flags, 1) + TypeDataFunCallable(int children_flags, std::vector&& params_types, TypePtr return_type) + : TypeData(children_flags) , params_types(std::move(params_types)) , return_type(return_type) {} @@ -289,21 +315,22 @@ class TypeDataFunCallable final : public TypeData { int params_size() const { return static_cast(params_types.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; /* - * `T` inside generic functions is TypeDataGenericT. + * `T` inside generic functions and structs is TypeDataGenericT. * Example: `fun f(a: X, b: Y): [X, Y]` (here X and Y are). + * Example: `struct Wrapper { value: T }` (type of field is generic T). * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. */ class TypeDataGenericT final : public TypeData { - TypeDataGenericT(uint64_t type_id, std::string&& nameT) - : TypeData(type_id, flag_contains_genericT_inside, -999999) // width undefined until instantiated + explicit TypeDataGenericT(std::string&& nameT) + : TypeData(flag_contains_genericT_inside) , nameT(std::move(nameT)) {} public: @@ -311,11 +338,68 @@ class TypeDataGenericT final : public TypeData { static TypePtr create(std::string&& nameT); + int get_width_on_stack() const override; + int get_type_id() const override; std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `Wrapper` when T is a generic (a struct is not ready to instantiate) is TypeDataGenericTypeWithTs. + * `Wrapper` is NOT here, it's an instantiated struct. Here is only when type arguments contain generics. + * Example: `type WrapperAlias = Wrapper`, then `Wrapper` (underlying type of alias) is here. + * Since structs and type aliases both can be generic, either struct_ref of alias_ref is filled. + */ +class TypeDataGenericTypeWithTs final : public TypeData { + TypeDataGenericTypeWithTs(int children_flags, StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) + : TypeData(children_flags) + , struct_ref(struct_ref) + , alias_ref(alias_ref) + , type_arguments(std::move(type_arguments)) {} + +public: + const StructPtr struct_ref; // for `Wrapper`, then alias_ref = nullptr + const AliasDefPtr alias_ref; // for `PairAlias`, then struct_ref = nullptr + const std::vector type_arguments; // ``, ``, at least one of them contains generic T + + static TypePtr create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments); + + int size() const { return static_cast(type_arguments.size()); } + + int get_width_on_stack() const override; + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `A`, `User`, `SomeStruct`, `Wrapper` is TypeDataStruct. At TVM level, structs are tensors. + * In the code, creating a struct is either `var v: A = { ... }` (by hint) or `var v = A { ... }`. + * Fields of a struct have their own types (accessed by struct_ref). + * Note, that instantiated structs like "Wrapper" exist in symtable (like aliases and functions), + * so for `Wrapper` struct_ref points to a generic struct, and for `Wrapper` to an instantiated one. + */ +class TypeDataStruct final : public TypeData { + explicit TypeDataStruct(StructPtr struct_ref) + : TypeData(0) + , struct_ref(struct_ref) {} + +public: + StructPtr struct_ref; + + static TypePtr create(StructPtr struct_ref); + + int get_width_on_stack() const override; + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. @@ -323,8 +407,8 @@ class TypeDataGenericT final : public TypeData { * A tensor can be empty. */ class TypeDataTensor final : public TypeData { - TypeDataTensor(uint64_t type_id, int children_flags, int width_on_stack, std::vector&& items) - : TypeData(type_id, children_flags, width_on_stack) + TypeDataTensor(int children_flags, std::vector&& items) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -334,22 +418,23 @@ class TypeDataTensor final : public TypeData { int size() const { return static_cast(items.size()); } + int get_width_on_stack() const override; + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; /* - * `[int, slice]` is TypeDataTypedTuple, a TVM 'tuple' under the hood, contained in 1 stack slot. + * `[int, slice]` is TypeDataBrackets, a TVM 'tuple' under the hood, contained in 1 stack slot. * Unlike TypeDataTuple (untyped tuples), it has a predefined inner structure and can be assigned as - * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). + * `var [i, cs] = [0, ""]` (where i and cs become two separate variables on a stack, int and slice). */ -class TypeDataTypedTuple final : public TypeData { - TypeDataTypedTuple(uint64_t type_id, int children_flags, std::vector&& items) - : TypeData(type_id, children_flags, 1) +class TypeDataBrackets final : public TypeData { + TypeDataBrackets(int children_flags, std::vector&& items) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -359,21 +444,45 @@ class TypeDataTypedTuple final : public TypeData { int size() const { return static_cast(items.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; /* - * `unknown` is a special type, which can appear in corner cases. - * The type of exception argument (which can hold any TVM value at runtime) is unknown. - * The type of `_` used as rvalue is unknown. - * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` + * `int8`, `int32`, `uint1`, `uint257`, `varint16` are TypeDataIntN. At TVM level, it's just int. + * The purpose of intN is to be used in struct fields, describing the way of serialization (n bits). + * A field `value: int32` has the TYPE `int32`, so being assigned to a variable, that variable is also `int32`. + * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. */ -class TypeDataUnknown final : public TypeData { - TypeDataUnknown() : TypeData(20ULL, flag_contains_unknown_inside, 1) {} +class TypeDataIntN final : public TypeData { + TypeDataIntN(int n_bits, bool is_unsigned, bool is_variadic) + : TypeData(0) + , n_bits(n_bits) + , is_unsigned(is_unsigned) + , is_variadic(is_variadic) {} + +public: + const int n_bits; + const bool is_unsigned; + const bool is_variadic; + + static TypePtr create(int n_bits, bool is_unsigned, bool is_variadic); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `coins` is just integer at TVM level, but encoded as varint when serializing structures. + * Example: `var cost = ton("0.05")` has type `coins`. + */ +class TypeDataCoins final : public TypeData { + TypeDataCoins() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -381,30 +490,113 @@ class TypeDataUnknown final : public TypeData { public: static TypePtr create() { return singleton; } - std::string as_human_readable() const override { return "unknown"; } + int get_type_id() const override { return 17; } + std::string as_human_readable() const override { return "coins"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; /* - * "Unresolved" is not actually a type — it's an intermediate state between parsing and resolving. - * At parsing to AST, unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, - * and after all source files parsed and global symbols registered, they are replaced by actual ones. - * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. + * `bits512`, `bytes8`, `bits9` are TypeDataBitsN. At TVM level, it's just slice. + * The purpose of bitsN is to be used in struct fields, describing the way of serialization (n bytes / n bits). + * In this essence, bitsN is very similar to intN. + * Note that unlike intN automatically cast to/from int, bitsN does NOT auto cast to slice (without `as`). */ -class TypeDataUnresolved final : public TypeData { - TypeDataUnresolved(uint64_t type_id, std::string&& text, SrcLocation loc) - : TypeData(type_id, flag_contains_unresolved_inside, -999999) - , text(std::move(text)) - , loc(loc) {} +class TypeDataBitsN final : public TypeData { + TypeDataBitsN(int n_width, bool is_bits) + : TypeData(0) + , n_width(n_width) + , is_bits(is_bits) {} public: - const std::string text; - const SrcLocation loc; + const int n_width; // either in bits, or in bytes + const bool is_bits; - static TypePtr create(std::string&& text, SrcLocation loc); + static TypePtr create(int n_width, bool is_bits); - std::string as_human_readable() const override { return text + "*"; } + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `T1 | T2 | ...` is a union type. Unions are supported in Tolk, similar to TypeScript, stored like enums in Rust. + * `T | null` (denoted as `T?`) is also a union from a type system point of view. + * There is no TypeDataNullable, because mixing unions and nullables would result in a mess. + * At TVM level: + * - `T | null`, if T is 1 slot (like `int | null`), then it's still 1 slot + * - `T | null`, if T is N slots (like `(int, int)?`), it's stored as N+1 slots (the last for type_id if T or 0 if null) + * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) + */ +class TypeDataUnion final : public TypeData { + TypeDataUnion(int children_flags, TypePtr or_null, std::vector&& variants) + : TypeData(children_flags) + , or_null(or_null) + , variants(std::move(variants)) {} + + bool has_variant_with_type_id(int type_id) const; + static void append_union_type_variant(TypePtr variant, std::vector& out_unique_variants); + +public: + const TypePtr or_null; // if `T | null`, then T is here (variants = [T, null] then); otherwise, nullptr + const std::vector variants; // T_i, flattened, no duplicates; may include aliases, but not other unions + + static TypePtr create(std::vector&& variants); + static TypePtr create_nullable(TypePtr nullable); + + int size() const { return static_cast(variants.size()); } + + // "primitive nullable" is `T?` which holds TVM NULL in the same slot (it other words, has no UTag slot) + // true : `int?`, `slice?`, `StructWith1IntField?` + // false: `(int, int)?`, `ComplexStruct?`, `()?` + bool is_primitive_nullable() const { + return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; + } + bool has_null() const { + if (or_null) { + return true; + } + return has_variant_with_type_id(0); + } + bool has_variant_with_type_id(TypePtr rhs_type) const { + int type_id = rhs_type->get_type_id(); + if (or_null) { + return type_id == 0 || type_id == or_null->get_type_id(); + } + return has_variant_with_type_id(type_id); + } + + TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; + bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + int get_variant_idx(TypePtr lookup_variant) const; + + int get_width_on_stack() const override; + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `unknown` is a special type, which can appear in corner cases. + * The type of exception argument (which can hold any TVM value at runtime) is unknown. + * The type of `_` used as rvalue is unknown. + * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` + */ +class TypeDataUnknown final : public TypeData { + TypeDataUnknown() : TypeData(flag_contains_unknown_inside) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + int get_type_id() const override; + std::string as_human_readable() const override { return "unknown"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; @@ -416,7 +608,7 @@ class TypeDataUnresolved final : public TypeData { * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. */ class TypeDataNever final : public TypeData { - TypeDataNever() : TypeData(19ULL, 0, 0) {} + TypeDataNever() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -424,6 +616,8 @@ class TypeDataNever final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; + int get_type_id() const override { return 19; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -436,7 +630,7 @@ class TypeDataNever final : public TypeData { * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. */ class TypeDataVoid final : public TypeData { - TypeDataVoid() : TypeData(10ULL, 0, 0) {} + TypeDataVoid() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -444,6 +638,8 @@ class TypeDataVoid final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; + int get_type_id() const override { return 10; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -453,11 +649,6 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- - -class Lexer; -TypePtr parse_type_from_tokens(Lexer& lex); -TypePtr parse_type_from_string(std::string_view text); - void type_system_init(); } // namespace tolk diff --git a/ton/ton-types.h b/ton/ton-types.h index c7aff6448..87947d027 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -62,7 +62,8 @@ enum GlobalCapabilities { capShortDequeue = 32, capStoreOutMsgQueueSize = 64, capMsgMetadata = 128, - capDeferMessages = 256 + capDeferMessages = 256, + capFullCollatedData = 512 }; inline int shard_pfx_len(ShardId shard) { @@ -410,6 +411,8 @@ struct Ed25519_PrivateKey { struct Ed25519_PublicKey { Bits256 _pubkey; + Ed25519_PublicKey() : _pubkey(td::Bits256::zero()) { + } explicit Ed25519_PublicKey(const Bits256& x) : _pubkey(x) { } explicit Ed25519_PublicKey(const td::ConstBitPtr x) : _pubkey(x) { @@ -444,14 +447,46 @@ struct Ed25519_PublicKey { }; // represents (the contents of) a block + +struct OutMsgQueueProofBroadcast : public td::CntObject { + OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs, + td::BufferSlice queue_proof, td::BufferSlice block_state_proof, int msg_count) + : dst_shard(std::move(dst_shard)) + , block_id(block_id) + , max_bytes(max_bytes) + , max_msgs(max_msgs) + , queue_proofs(std::move(queue_proof)) + , block_state_proofs(std::move(block_state_proof)) + , msg_count(std::move(msg_count)) { + } + ShardIdFull dst_shard; + BlockIdExt block_id; + + // importedMsgQueueLimits + td::uint32 max_bytes; + td::uint32 max_msgs; + + // outMsgQueueProof + td::BufferSlice queue_proofs; + td::BufferSlice block_state_proofs; + int msg_count; + + OutMsgQueueProofBroadcast* make_copy() const override { + return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(), + block_state_proofs.clone(), msg_count); + } +}; + struct BlockCandidate { BlockCandidate(Ed25519_PublicKey pubkey, BlockIdExt id, FileHash collated_file_hash, td::BufferSlice data, - td::BufferSlice collated_data) + td::BufferSlice collated_data, + std::vector> out_msg_queue_broadcasts = {}) : pubkey(pubkey) , id(id) , collated_file_hash(collated_file_hash) , data(std::move(data)) - , collated_data(std::move(collated_data)) { + , collated_data(std::move(collated_data)) + , out_msg_queue_proof_broadcasts(std::move(out_msg_queue_broadcasts)) { } Ed25519_PublicKey pubkey; BlockIdExt id; @@ -459,11 +494,32 @@ struct BlockCandidate { td::BufferSlice data; td::BufferSlice collated_data; + // used only locally + std::vector> out_msg_queue_proof_broadcasts; + BlockCandidate clone() const { - return BlockCandidate{pubkey, id, collated_file_hash, data.clone(), collated_data.clone()}; + return BlockCandidate{ + pubkey, id, collated_file_hash, data.clone(), collated_data.clone(), out_msg_queue_proof_broadcasts}; + } +}; + +struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + GeneratedCandidate clone() const { + return {candidate.clone(), is_cached, self_collated, collator_node_id}; } }; +struct BlockCandidatePriority { + td::uint32 round{}; + td::uint32 first_block_round{}; + td::int32 priority{}; +}; + struct ValidatorDescr { /* ton::validator::ValidatorFullId */ Ed25519_PublicKey key; ValidatorWeight weight; @@ -515,8 +571,13 @@ struct ValidatorSessionConfig { }; struct PersistentStateDescription : public td::CntObject { + struct ShardBlock { + BlockIdExt block; + td::uint32 split_depth; + }; + BlockIdExt masterchain_id; - std::vector shard_blocks; + std::vector shard_blocks; UnixTime start_time, end_time; virtual CntObject* make_copy() const { diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 3dbd628d3..f28176223 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) if (NOT OPENSSL_FOUND) @@ -74,14 +72,14 @@ if (TONLIB_ENABLE_JNI AND NOT ANDROID) # jni is available by default on Android endif() add_executable(tonlib-cli tonlib/tonlib-cli.cpp) -target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal pow-miner-lib git) +target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal ton_crypto git) if (NOT CMAKE_CROSSCOMPILING) if (TONLIB_ENABLE_JNI) #FIXME #add_dependencies(tonlib tonlib_generate_java_api) endif() -endif() +endif() add_library(tonlibjson_private STATIC tonlib/ClientJson.cpp tonlib/ClientJson.h) target_include_directories(tonlibjson_private PUBLIC $ @@ -104,6 +102,17 @@ else() target_link_libraries(tonlibjson PRIVATE tonlibjson_private) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index e972d84e0..45d6c9ebb 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -93,8 +93,9 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); - TRY_RESULT(config, block::ConfigInfo::extract_config( - std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); + TRY_RESULT(config, + block::ConfigInfo::extract_config( + std::move(state), blkid, block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d73e715c9..87f285a36 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -180,7 +180,7 @@ struct RawAccountState { td::Ref extra_currencies; ton::UnixTime storage_last_paid{0}; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_used; td::Ref code; td::Ref data; @@ -268,7 +268,7 @@ td::Result> add_extra_currencies(const td::Ref &e1, block::CurrencyCollection c1{td::zero_refint(), e1}; block::CurrencyCollection c2{td::zero_refint(), e2}; TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); - if (c1.is_valid()) { + if (!c1.is_valid()) { return td::Status::Error("Failed to add extra currencies"); } return c1.extra; @@ -1036,7 +1036,7 @@ class Query { TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( - raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, + raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used, raw_.source->raw().storage_last_paid, false, is_masterchain); auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long(); @@ -1085,7 +1085,7 @@ class Query { TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( - destination->get_sync_time(), storage_prices, destination->raw().storage_stat, + destination->get_sync_time(), storage_prices, destination->raw().storage_used, destination->raw().storage_last_paid, false, is_masterchain) : td::make_refint(0); Fee dst_fee; @@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor { return td::Status::Error("Failed to unpack StorageInfo"); } unsigned long long u = 0; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_stat; u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells); u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells); //LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells - //<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells; + //<< " bits=" << storage_stat.bits; if (u == std::numeric_limits::max()) { return td::Status::Error("Failed to unpack StorageStat"); } - res.storage_stat = storage_stat; + res.storage_used = storage_stat; } block::gen::AccountStorage::Record storage; @@ -1802,6 +1801,132 @@ class GetShardBlockProof : public td::actor::Actor { std::vector> links_; }; +class GetOutMsgQueueSizes : public td::actor::Actor { + public: + GetOutMsgQueueSizes(ExtClientRef ext_client_ref, std::vector blocks, + td::actor::ActorShared<> parent, + td::Promise>&& promise) + : blocks_(std::move(blocks)), parent_(std::move(parent)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + sizes_.resize(blocks_.size()); + pending_ = blocks_.size() + 1; + + for (size_t i = 0; i < blocks_.size(); ++i) { + client_.send_query( + ton::lite_api::liteServer_getBlockOutMsgQueueSize(1, ton::create_tl_lite_block_id(blocks_[i]), true), + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_block_queue_size, i, R.move_as_ok()); + } + }); + } + + client_.send_query( + ton::lite_api::liteServer_getOutMsgQueueSizes(1, ton::masterchainId, ton::shardIdAll), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_ext_msg_queue_size_limit, + R.ok()->ext_msg_queue_size_limit_); + } + }); + } + + void got_block_queue_size(size_t i, lite_api_ptr f) { + try { + auto S = [&, this]() -> td::Status { + TRY_RESULT_PREFIX(roots, vm::std_boc_deserialize_multi(f->proof_), "cannot deserialize proof: "); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in proof"); + } + auto state_root = vm::MerkleProof::virtualize(std::move(roots[1]), 1); + if (state_root.is_null()) { + return td::Status::Error("state proof is invalid"); + } + ton::Bits256 state_hash = state_root->get_hash().bits(); + TRY_STATUS_PREFIX(block::check_block_header_proof(vm::MerkleProof::virtualize(std::move(roots[0]), 1), + blocks_[i], &state_hash, true, nullptr, nullptr), + "error in block header proof: "); + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + return td::Status::Error("cannot unpack shard state"); + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + return td::Status::Error("cannot unpack OutMsgQueueExtra"); + } + vm::CellSlice& size_slice = out_msg_queue_extra.out_queue_size.write(); + if (size_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + td::uint64 size = size_slice.prefetch_ulong(48); + if (size != f->size_) { + return td::Status::Error("queue size mismatch"); + } + return td::Status::OK(); + }(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + } catch (vm::VmError& err) { + abort(err.as_status()); + return; + } catch (vm::VmVirtError& err) { + abort(err.as_status()); + return; + } + + sizes_[i] = f->size_; + dec_pending(); + } + + void got_ext_msg_queue_size_limit(td::uint32 value) { + ext_msg_queue_size_limit_ = value; + dec_pending(); + } + + void dec_pending() { + if (--pending_ == 0) { + std::vector> shards; + for (size_t i = 0; i < blocks_.size(); ++i) { + shards.push_back( + tonlib_api::make_object(to_tonlib_api(blocks_[i]), sizes_[i])); + } + promise_.set_result( + tonlib_api::make_object(std::move(shards), ext_msg_queue_size_limit_)); + stop(); + } + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + std::vector blocks_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + + std::vector sizes_; + td::uint32 ext_msg_queue_size_limit_ = 0; + size_t pending_ = 0; +}; + auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; @@ -2043,7 +2168,7 @@ class RunEmulator : public TonlibQueryActor { } try { - auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, block_id_.mc, 0b11'11111111); if (r_config.is_error()) { check(r_config.move_as_error()); return; @@ -2089,7 +2214,7 @@ class RunEmulator : public TonlibQueryActor { raw.balance = balance.grams->to_long(); raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); - raw.storage_stat = std::move(account.storage_stat); + raw.storage_used = account.storage_used; raw.code = std::move(account.code); raw.data = std::move(account.data); raw.state = std::move(account.total_state); @@ -3100,15 +3225,12 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); - TRY_RESULT(ihr_fee, to_balance(msg_info.ihr_fee)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, - std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), - get_data(src)); + msg_hash, tonlib_api::make_object(src), + tonlib_api::make_object(std::move(dest)), balance, std::move(extra_currencies), + fwd_fee, /* ihr_fee = */ 0, created_lt, std::move(body_hash), get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; @@ -3308,13 +3430,53 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessage& request, return td::Status::OK(); } +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell) { + block::gen::Message::Record message; + if (!tlb::type_unpack_cell(ext_in_msg_cell, block::gen::t_Message_Any, message)) { + return td::Status::Error("Failed to unpack Message"); + } + auto tag = block::gen::CommonMsgInfo().get_tag(*message.info); + if (tag != block::gen::CommonMsgInfo::ext_in_msg_info) { + return td::Status::Error("CommonMsgInfo tag is not ext_in_msg_info"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::csr_unpack(message.info, msg_info)) { + return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); + } + + td::Ref body; + auto body_cs = message.body.write(); + if (body_cs.fetch_ulong(1) == 1) { + body = body_cs.fetch_ref(); + } else { + body = vm::CellBuilder().append_cellslice(body_cs).finalize(); + } + + auto cb = vm::CellBuilder(); + bool status = + cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 + cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 + cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt + cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 + cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 + cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 + cb.store_ref_bool(body); + + if (!status) { + return td::Status::Error("Failed to build normalized message"); + } + return cb.finalize()->get_hash().bits(); +} + td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& request, td::Promise>&& promise) { TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); auto hash = body->get_hash().as_slice().str(); + TRY_RESULT(hash_norm, get_ext_in_msg_hash_norm(body)); + make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash)](auto res) { - return tonlib_api::make_object(std::move(hash)); + promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { + return tonlib_api::make_object(std::move(hash), hash_norm.as_slice().str()); })); return td::Status::OK(); } @@ -4510,7 +4672,7 @@ auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result& stack) -> td::Result>> { - int stack_limit = 1000; + int stack_limit = 8000; std::vector> tl_stack; for (auto& entry: stack->as_span()) { TRY_RESULT(tl_entry, to_tonlib_api(entry, --stack_limit)); @@ -5004,7 +5166,7 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in return; } - TRY_RESULT_PROMISE(promise, args, ton::DnsInterface::resolve_args(name, category, address)); + TRY_RESULT_PROMISE(new_promise, args, ton::DnsInterface::resolve_args(name, category, address)); int_api::RemoteRunSmcMethod query; query.address = std::move(address); query.args = std::move(args); @@ -6138,19 +6300,34 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& td::Status TonlibClient::do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, td::Promise>&& promise) { - client_.send_query(ton::lite_api::liteServer_getOutMsgQueueSizes(request.mode_, request.wc_, request.shard_), - promise.wrap([](lite_api_ptr&& queue_sizes) { - tonlib_api::blocks_outMsgQueueSizes result; - result.ext_msg_queue_size_limit_ = queue_sizes->ext_msg_queue_size_limit_; - for (auto &x : queue_sizes->shards_) { - tonlib_api::blocks_outMsgQueueSize shard; - shard.id_ = to_tonlib_api(*x->id_); - shard.size_ = x->size_; - result.shards_.push_back(tonlib_api::make_object(std::move(shard))); - } - return tonlib_api::make_object(std::move(result)); - })); - + auto req_mode = request.mode_; + auto req_shard = ton::ShardIdFull{request.wc_, (ton::ShardId)request.shard_}; + if ((req_mode & 1) && !req_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + client_.with_last_block( + [=, self = this, promise = std::move(promise)](td::Result r_last_block) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, last_block, std::move(r_last_block), "get last block failed: "); + do_request(tonlib_api::blocks_getShards(to_tonlib_api(last_block.last_block_id)), + [=, mc_blkid = last_block.last_block_id, + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, shards, std::move(R), "get shards failed: "); + std::vector blocks; + if (!(req_mode & 1) || ton::shard_intersects(mc_blkid.shard_full(), req_shard)) { + blocks.push_back(mc_blkid); + } + for (const auto& shard : shards->shards_) { + TRY_RESULT_PROMISE(promise, block_id, to_block_id(*shard)); + if (!(req_mode & 1) || ton::shard_intersects(block_id.shard_full(), req_shard)) { + blocks.push_back(block_id); + } + } + auto actor_id = self->actor_id_++; + self->actors_[actor_id] = td::actor::create_actor( + "GetOutMsgQueueSizes", self->client_.get_client(), std::move(blocks), + actor_shared(this, actor_id), std::move(promise)); + }); + }); return td::Status::OK(); } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 28239a8a4..ed19be65e 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -59,6 +59,7 @@ class RunEmulator; td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data); td::Result to_dns_entry_data(tonlib_api::dns_EntryData& entry_data); +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell); class TonlibClient : public td::actor::Actor { public: diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 2c7100f24..3a7d72e52 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -52,7 +52,6 @@ #include "auto/tl/tonlib_api.hpp" -#include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" #include "lite-client/ext-client.h" @@ -329,11 +328,6 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "rwallet address \n"; td::TerminalIO::out() << "rwallet init [: ...]\n"; } - void pminer_help() { - td::TerminalIO::out() << "pminer help\n"; - td::TerminalIO::out() << "pminer start \n"; - td::TerminalIO::out() << "pminer stop\n"; - } void parse_line(td::BufferSlice line) { if (is_closing_) { @@ -393,7 +387,6 @@ class TonlibCli : public td::actor::Actor { dns_help(); pchan_help(); rwallet_help(); - pminer_help(); td::TerminalIO::out() << "blockmode auto|manual\tWith auto mode, all queries will be executed with respect to the latest block. " @@ -430,6 +423,7 @@ class TonlibCli : public td::actor::Actor { << "\t 'k' modifier - use fake key\n" << "\t 'c' modifier - just esmitate fees\n"; td::TerminalIO::out() << "getmasterchainsignatures - get sigratures of masterchain block \n"; + td::TerminalIO::out() << "msgqueuesizes - get out msg queue sizes in the latest shard states\n"; } else if (cmd == "genkey") { generate_key(); } else if (cmd == "exit" || cmd == "quit") { @@ -504,8 +498,6 @@ class TonlibCli : public td::actor::Actor { run_pchan_cmd(parser, std::move(cmd_promise)); } else if (cmd == "rwallet") { run_rwallet_cmd(parser, std::move(cmd_promise)); - } else if (cmd == "pminer") { - run_pminer_cmd(parser, std::move(cmd_promise)); } else if (cmd == "gethistory") { get_history(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "guessrevision") { @@ -517,6 +509,8 @@ class TonlibCli : public td::actor::Actor { } else if (cmd == "getmasterchainsignatures") { auto seqno = parser.read_word(); run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); + } else if (cmd == "msgqueuesizes") { + run_get_out_msg_queue_sizes(std::move(cmd_promise)); } else if (cmd == "showtransactions") { run_show_transactions(parser, std::move(cmd_promise)); } else { @@ -590,216 +584,6 @@ class TonlibCli : public td::actor::Actor { promise.set_error(td::Status::Error("Unknown command")); } - class PowMiner : public td::actor::Actor { - public: - class Callback { - public: - }; - struct Options { - Address giver_address; - Address my_address; - }; - - PowMiner(Options options, td::actor::ActorId client) - : options_(std::move(options)), client_(std::move(client)) { - } - - private: - Options options_; - td::actor::ActorId client_; - - td::optional miner_options_; - static constexpr double QUERY_EACH = 5.0; - td::Timestamp next_options_query_at_; - - bool need_run_miners_{false}; - td::CancellationTokenSource source_; - ton::Miner::Options miner_options_copy_; - std::size_t threads_alive_{0}; - std::vector threads_; - - bool close_flag_{false}; - - template - void send_query(QueryT query, td::Promise promise) { - td::actor::send_lambda(client_, - [self = client_, query = std::move(query), promise = std::move(promise)]() mutable { - self.get_actor_unsafe().make_request(std::move(query), std::move(promise)); - }); - } - - void start_up() override { - next_options_query_at_ = td::Timestamp::now(); - loop(); - } - void hangup() override { - close_flag_ = true; - source_.cancel(); - try_stop(); - } - void try_stop() { - if (threads_alive_ == 0) { - td::TerminalIO::out() << "pminer: stopped\n"; - stop(); - } - } - - void loop() override { - if (close_flag_) { - try_stop(); - return; - } - if (next_options_query_at_ && next_options_query_at_.is_in_past()) { - send_query(tonlib_api::smc_load(options_.giver_address.tonlib_api()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_state)); - next_options_query_at_ = {}; - } - - if (miner_options_ && threads_.empty() && need_run_miners_) { - td::TerminalIO::out() << "pminer: start workers\n"; - need_run_miners_ = false; - miner_options_copy_ = miner_options_.value(); - miner_options_copy_.token_ = source_.get_cancellation_token(); - auto n = td::thread::hardware_concurrency(); - threads_alive_ = n; - for (td::uint32 i = 0; i < n; i++) { - threads_.emplace_back([this, actor_id = actor_id(this)] { - auto res = ton::Miner::run(miner_options_copy_); - global_scheduler_->run_in_context_external( - [&] { send_closure(actor_id, &PowMiner::got_answer, std::move(res)); }); - }); - } - } - - alarm_timestamp().relax(next_options_query_at_); - } - - void got_answer(td::optional answer) { - source_.cancel(); - if (--threads_alive_ == 0) { - threads_.clear(); - } - if (answer) { - td::TerminalIO::out() << "pminer: got some result - sending query to the giver\n"; - vm::CellBuilder cb; - cb.store_bytes(answer.unwrap()); - send_query(tonlib_api::raw_createAndSendMessage( - options_.giver_address.tonlib_api(), "", - vm::std_boc_serialize(cb.finalize_novm()).move_as_ok().as_slice().str()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::on_query_sent)); - } - loop(); - } - - void on_query_sent(td::Result> r_ok) { - LOG_IF(ERROR, r_ok.is_error()) << "pminer: " << r_ok.error(); - } - - void with_giver_state(td::Result> r_info) { - if (r_info.is_error()) { - return with_giver_info(r_info.move_as_error()); - } - send_query(tonlib_api::smc_runGetMethod(r_info.ok()->id_, - make_object("get_pow_params"), {}), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_info)); - } - - void with_giver_info(td::Result> r_info) { - auto status = do_with_giver_info(std::move(r_info)); - LOG_IF(ERROR, status.is_error()) << "pminer: " << status; - next_options_query_at_ = td::Timestamp::in(QUERY_EACH); - return loop(); - } - - td::Result to_number(const tonlib_api::object_ptr& entry, - td::int32 bits) { - if (entry->get_id() != tonlib_api::tvm_stackEntryNumber::ID) { - return td::Status::Error("Expected stackEntryNumber"); - } - auto& number_str = static_cast(*entry.get()).number_->number_; - auto num = td::make_refint(); - if (num.write().parse_dec(number_str.data(), (int)number_str.size()) < (int)number_str.size()) { - return td::Status::Error("Failed to parse a number"); - } - if (!num->unsigned_fits_bits(bits)) { - return td::Status::Error(PSLICE() << "Number is too big " << num->to_dec_string() << " " << bits); - } - return num; - } - - td::Status do_with_giver_info(td::Result> r_info) { - TRY_RESULT(info, std::move(r_info)); - if (info->stack_.size() < 2) { - return td::Status::Error("Unexpected `get_pow_params` result format"); - } - TRY_RESULT(seed, to_number(info->stack_[0], 128)); - TRY_RESULT(complexity, to_number(info->stack_[1], 256)); - ton::Miner::Options options; - seed->export_bytes(options.seed.data(), 16, false); - complexity->export_bytes(options.complexity.data(), 32, false); - - TRY_RESULT(address, block::StdAddress::parse(options_.my_address.address->account_address_)); - options.my_address = std::move(address); - options.token_ = source_.get_cancellation_token(); - - if (miner_options_ && miner_options_.value().seed == options.seed) { - return td::Status::OK(); - } - - td::TerminalIO::out() << "pminer: got new options\n"; - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*complexity, hrate); - long long hash_rate = hrate.to_long(); - td::TerminalIO::out() << "[ expected required hashes for success: " << hash_rate << " ]\n"; - miner_options_ = std::move(options); - need_run_miners_ = true; - source_.cancel(); - - return td::Status::OK(); - } - }; - - td::uint64 pow_miner_id_{0}; - std::map> pow_miners_; - - void pminer_start(td::ConstParser& parser, td::Promise promise) { - if (!pow_miners_.empty()) { - promise.set_error(td::Status::Error("One pminer is already running")); - } - TRY_RESULT_PROMISE_PREFIX(promise, giver_address, to_account_address(parser.read_word(), false), "giver address"); - TRY_RESULT_PROMISE_PREFIX(promise, my_address, to_account_address(parser.read_word(), false), "my address"); - - auto id = ++pow_miner_id_; - - PowMiner::Options options; - options.giver_address = std::move(giver_address); - options.my_address = std::move(my_address); - pow_miners_.emplace(id, td::actor::create_actor("PowMiner", std::move(options), client_.get())); - td::TerminalIO::out() << "Miner #" << id << " created"; - promise.set_value({}); - } - - void pminer_stop(td::ConstParser& parser, td::Promise promise) { - pow_miners_.clear(); - promise.set_value({}); - } - - void run_pminer_cmd(td::ConstParser& parser, td::Promise promise) { - auto cmd = parser.read_word(); - if (cmd == "help") { - pminer_help(); - return promise.set_value(td::Unit()); - } - - if (cmd == "start") { - return pminer_start(parser, std::move(promise)); - } - if (cmd == "stop") { - return pminer_stop(parser, std::move(promise)); - } - promise.set_error(td::Status::Error("Unknown command")); - } - void run_pchan_cmd(td::ConstParser& parser, td::Promise promise) { auto cmd = parser.read_word(); if (cmd == "help") { @@ -2161,6 +1945,22 @@ class TonlibCli : public td::actor::Actor { })); } + void run_get_out_msg_queue_sizes(td::Promise promise) { + send_query(make_object(0, 0, 0), + promise.wrap([](tonlib_api::object_ptr&& f) { + td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; + for (const auto& shard : f->shards_) { + td::TerminalIO::out() << ton::BlockId{shard->id_->workchain_, (ton::ShardId)shard->id_->shard_, + (ton::BlockSeqno)shard->id_->seqno_} + .to_str() + << " " << shard->size_ << std::endl; + } + td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ + << std::endl; + return td::Unit(); + })); + } + void run_show_transactions(td::ConstParser& parser, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); TRY_RESULT_PROMISE(promise, lt, td::to_integer_safe(parser.read_word())); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e9ffd2a27..0afdf7389 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -24,4 +22,7 @@ target_include_directories(pack-viewer PUBLIC $. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" +#include "ton/lite-tl.hpp" + +#include "td/utils/overloaded.h" + +#include +#include "td/utils/tl_storers.h" +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" + +#include +#include "block/block-auto.h" +#include "block/mc-config.h" + +using namespace ton; + +std::string global_config_file; +td::optional start_mc_seqno, end_mc_seqno; +std::vector shards; + +class PrepareLsSliceConfig : public td::actor::Actor { + public: + void start_up() override { + if (start_mc_seqno && end_mc_seqno && start_mc_seqno.value() > end_mc_seqno.value()) { + LOG(FATAL) << "from-seqno is greater than to-seqno"; + } + + if (!start_mc_seqno && !end_mc_seqno) { + auto slice = create_tl_object(); + if (shards.empty()) { + slice->shards_.push_back(create_tl_shard_id(ShardIdFull{basechainId, shardIdAll})); + } else { + for (const ShardIdFull& shard : shards) { + if (!shard.is_masterchain()) { + slice->shards_.push_back(create_tl_shard_id(shard)); + } + } + } + print_result(*slice); + return; + } + + auto gc_s = td::read_file(global_config_file).move_as_ok(); + auto gc_j = td::json_decode(gc_s.as_slice()).move_as_ok(); + ton_api::liteclient_config_global gc; + ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + client_ = liteclient::ExtClient::create(r_servers.move_as_ok(), nullptr); + + slice_timed_ = create_tl_object(); + ++pending_; + request_shards_info(start_mc_seqno, true); + request_shards_info(end_mc_seqno, false); + dec_pending(); + } + + template + static td::BufferSlice create_query(Args&&... args) { + Type object(std::forward(args)...); + return create_serialize_tl_object(serialize_tl_object(&object, true)); + } + + template + static tl_object_ptr parse_response(const td::Result& R) { + R.ensure(); + auto err = fetch_tl_object(R.ok(), true); + if (err.is_ok()) { + LOG(FATAL) << "liteserver error: " << err.ok()->message_; + } + auto res = fetch_tl_object(R.ok(), true); + res.ensure(); + return res.move_as_ok(); + } + + void request_shards_info(td::optional seqno, bool is_start) { + if (!seqno) { + return; + } + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query( + 1, create_tl_object(masterchainId, shardIdAll, seqno.value()), 0, 0), + td::Timestamp::in(5.0), [=, client = client_.get(), SelfId = actor_id(this)](td::Result R) { + auto mc_header = parse_response(std::move(R)); + auto block_id = create_block_id(mc_header->id_); + td::actor::send_closure( + client, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id)), + td::Timestamp::in(5.0), [=, mc_header = std::move(mc_header)](td::Result R) mutable { + auto shards_info = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_shards_info, std::move(mc_header), + std::move(shards_info), is_start); + }); + }); + } + + static tl_object_ptr parse_header(const lite_api::liteServer_blockHeader& obj, + bool is_start) { + auto res = create_tl_object(); + + BlockIdExt block_id = create_block_id(obj.id_); + res->shard_id_ = create_tl_shard_id(block_id.shard_full()); + res->seqno_ = block_id.seqno(); + + auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); + root = vm::MerkleProof::virtualize(root, 1); + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); + res->utime_ = info.gen_utime; + res->lt_ = (is_start ? info.start_lt : info.end_lt); + + return res; + } + + void got_shards_info(tl_object_ptr mc_header, + tl_object_ptr shards_info, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*mc_header, is_start)); + + auto root = vm::std_boc_deserialize(shards_info->data_).move_as_ok(); + block::ShardConfig sh_conf; + CHECK(sh_conf.unpack(vm::load_cell_slice_ref(root))); + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + BlockIdExt block_id = sh_conf.get_shard_hash(ton::ShardIdFull(id))->top_block_id(); + bool ok = shards.empty(); + for (const auto& our_shard : shards) { + if (shard_intersects(our_shard, block_id.shard_full())) { + ok = true; + break; + } + } + if (ok) { + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id), 0xffff), + td::Timestamp::in(5.0), [=, SelfId = actor_id(this)](td::Result R) mutable { + auto header = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_block_header, std::move(header), is_start); + }); + } + } + + dec_pending(); + } + + void got_block_header(tl_object_ptr header, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*header, is_start)); + dec_pending(); + } + + void print_result(const ton_api::liteserver_descV2_Slice& result) { + auto s = td::json_encode(td::ToJson(result), true); + std::cout << s << "\n"; + std::cout.flush(); + exit(0); + } + + private: + td::actor::ActorOwn client_; + tl_object_ptr slice_timed_; + size_t pending_ = 0; + + void dec_pending() { + --pending_; + if (pending_ == 0) { + auto cmp = [](const tl_object_ptr& a, + const tl_object_ptr& b) { + return create_shard_id(a->shard_id_) < create_shard_id(b->shard_id_); + }; + std::sort(slice_timed_->shards_from_.begin(), slice_timed_->shards_from_.end(), cmp); + std::sort(slice_timed_->shards_to_.begin(), slice_timed_->shards_to_.end(), cmp); + print_result(*slice_timed_); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + td::OptionParser p; + p.set_description( + "Generate liteserver.descV2.Slice for global-config.json from given shards and masterchain seqnos\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "prepare-ls-slice-config build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('C', "global-config", "global TON configuration file (used to fetch shard configuration)", + [&](td::Slice arg) { global_config_file = arg.str(); }); + p.add_checked_option('f', "from-seqno", "starting masterchain seqno (default: none)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(start_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('t', "to-seqno", "ending masterchain seqno (default: none)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(end_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('s', "shard", "shard in format 0:8000000000000000 (default: all shards)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT(shard, ShardIdFull::parse(arg)); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << arg); + } + shards.push_back(shard); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({3}); + + scheduler.run_in_context([&] { td::actor::create_actor("main").release(); }); + while (scheduler.run(1)) { + } +} diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index a9baa7595..e89cb1bbf 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -184,7 +184,7 @@ class ProxyLiteserver : public td::actor::Actor { for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), i)); server.alive = false; } @@ -197,7 +197,7 @@ class ProxyLiteserver : public td::actor::Actor { } server.alive = ready; LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; } void create_ext_server() { @@ -295,8 +295,7 @@ class ProxyLiteserver : public td::actor::Actor { Server& server = servers_[server_idx]; LOG(INFO) << "Sending query " << query_info.to_str() << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") - << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.hostname << ")"; BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); diff --git a/validator-engine-console/CMakeLists.txt b/validator-engine-console/CMakeLists.txt index d87d8a6f3..73f3a5877 100644 --- a/validator-engine-console/CMakeLists.txt +++ b/validator-engine-console/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable (validator-engine-console validator-engine-console.cpp validator-engine-console.h validator-engine-console-query.cpp validator-engine-console-query.h ) diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d11100194..1a9ca052e 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -807,21 +807,21 @@ td::Status SignCertificateQuery::send() { auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); auto pub = ton::create_serialize_tl_object(signer_.tl()); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); return td::Status::OK(); } @@ -833,33 +833,32 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { } pubkey_ = f.move_as_ok(); has_pubkey_ = true; - if(has_signature_) { + if (has_signature_) { save_certificate(); } } - td::Status SignCertificateQuery::receive(td::BufferSlice data) { UNREACHABLE(); } void SignCertificateQuery::receive_signature(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); - if(f.is_error()){ + if (f.is_error()) { handle_error(f.move_as_error_prefix("Failed to get signature: ")); return; } signature_ = std::move(f.move_as_ok()->signature_); - if(has_pubkey_) { + if (has_pubkey_) { save_certificate(); } } void SignCertificateQuery::save_certificate() { - auto c = ton::create_serialize_tl_object( - std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto c = ton::create_serialize_tl_object(std::move(pubkey_), expire_at_, max_size_, + std::move(signature_)); auto w = td::write_file(out_file_, c.as_slice()); - if(w.is_error()) { + if (w.is_error()) { handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); return; } @@ -880,11 +879,8 @@ td::Status ImportCertificateQuery::send() { TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); auto b = ton::create_serialize_tl_object( - overlay_, - ton::create_tl_object(id_), - ton::create_tl_object(kh_.tl()), - std::move(cert) - ); + overlay_, ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -920,6 +916,10 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { << "\n"; sb << " is_neighbour: " << n->is_neighbour_ << " is_alive: " << n->is_alive_ << " node_flags: " << n->node_flags_ << "\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " last_ping_at: " << (td::uint32)n->last_ping_at_ << " (" << time_to_human((td::uint32)n->last_ping_at_) + << ") last_ping_time: " << n->last_ping_time_ << "\n"; + } print_traffic("throughput", " ", n->traffic_); print_traffic("throughput (responses only)", " ", n->traffic_responses_); } @@ -985,8 +985,12 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" - << time_to_human(n->last_out_query_) << "\",\n" - << "\n "; + << time_to_human(n->last_out_query_) << "\",\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " \"last_ping_at\": " << (td::uint32)n->last_ping_at_ << ", \"last_ping_at_human\": \"" + << time_to_human((td::uint32)n->last_ping_at_) << "\", \"last_ping_time\": " << n->last_ping_time_ << ",\n"; + } + sb << "\n "; print_traffic("throughput", n->traffic_); sb << ",\n "; print_traffic("throughput_responses", n->traffic_responses_); @@ -1031,7 +1035,6 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status ImportCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); @@ -1039,7 +1042,6 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status SignShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); @@ -1062,7 +1064,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), "received incorrect cert: "); auto w = td::write_file(out_file_, data.as_slice()); - if(w.is_error()) { + if (w.is_error()) { return w.move_as_error_prefix("Failed to write certificate to file: "); } td::TerminalIO::out() << "saved certificate\n"; @@ -1292,6 +1294,9 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; } } + if (overlay->skip_public_msg_send_) { + td::TerminalIO::out() << "Don't send external messages to public overlays\n"; + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); @@ -1595,4 +1600,429 @@ td::Status DelShardQuery::receive(td::BufferSlice data) { "received incorrect answer: "); td::TerminalIO::out() << "successfully removed shard\n"; return td::Status::OK(); -} \ No newline at end of file +} + +td::Status AddCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_.tl(), + ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; + return td::Status::OK(); +} + +td::Status DelCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_.tl(), + ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), true); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), false); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collator node whitelist: " << (f->enabled_ ? "ENABLED" : "DISABLED") << "\n"; + td::TerminalIO::out() << f->adnl_ids_.size() << " validator adnl ids\n"; + for (const auto &id : f->adnl_ids_) { + td::TerminalIO::out() << id.to_hex() << "\n"; + } + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collators list:\n"; + if (list->shards_.empty()) { + td::TerminalIO::out() << "Shard list is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : list->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; + for (const auto &collator : shard->collators_) { + td::TerminalIO::out() << " Collator " << collator->adnl_id_ << "\n"; + } + } + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + if (list->local_ids_.empty()) { + td::TerminalIO::out() << "No stats\n"; + return td::Status::OK();; + } + for (auto &stats : list->local_ids_) { + td::TerminalIO::out() << "VALIDATOR ADNL ID = " << stats->adnl_id_ << "\n"; + std::map collators; + for (auto &collator: stats->collators_) { + collators[collator->adnl_id_] = collator.get(); + } + for (auto &shard : stats->shards_) { + td::TerminalIO::out() << " Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; + td::TerminalIO::out() << " Active = " << shard->active_ << "\n"; + td::TerminalIO::out() << " Collators: " << shard->collators_.size() << "\n"; + for (auto &id : shard->collators_) { + auto collator = collators[id]; + if (collator == nullptr) { + return td::Status::Error("collator not found"); + } + td::StringBuilder sb; + sb << " " << id << "\n"; + sb << " alive=" << (int)collator->alive_; + if (collator->active_) { + sb << " ping_in=" << td::StringBuilder::FixedDouble(std::max(collator->ping_in_, 0.0), 3); + } + sb << " last_ping_ago="; + if (collator->last_ping_ago_ < 0.0) { + sb << "never"; + } else { + std::string status = collator->last_ping_status_; + std::erase_if(status, [](char c) { return c < (char)32; }); + if (status.size() > 128) { + status.resize(128); + } + sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; + } + if (collator->banned_for_ > 0.0) { + sb << " banned_for=" << td::StringBuilder::FixedDouble(std::max(collator->banned_for_, 0.0), 3); + } + td::TerminalIO::out() << sb.as_cslice() << "\n"; + } + } + } + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::send() { + auto b = ton::create_serialize_tl_object( + key_hash_, adnl_id_, slot_, expire_at_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS(td::write_file(file_name_, data)); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(certificate, ton::fetch_tl_object(data, true)); + auto b = ton::create_serialize_tl_object( + adnl_id_, std::move(certificate)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_, slot_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::send() { + auto list = ton::create_tl_object(); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + config, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Shard block verifier config:\n"; + if (config->shards_.empty()) { + td::TerminalIO::out() << "Config is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : config->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Required confirms = " << shard->required_confirms_ << "/" + << shard->trusted_nodes_.size() << "\n"; + td::TerminalIO::out() << " Trusted nodes:\n"; + for (const auto &node : shard->trusted_nodes_) { + td::TerminalIO::out() << " " << node << "\n"; + } + } + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 817d70c9e..c06924d9a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -151,8 +151,8 @@ inline td::Result Tokenizer::get_token() { auto r_wc = td::to_integer_safe(word); if (r_wc.is_ok()) { TRY_RESULT_ASSIGN(word, get_raw_token()); - TRY_RESULT(shard, td::to_integer_safe(word)); - return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), (ton::ShardId)shard}; } return ton::ShardIdFull::parse(word); } @@ -990,9 +990,9 @@ class GetOverlaysStatsJsonQuery : public Query { std::string name() const override { return get_name(); } - -private: - std::string file_name_; + + private: + std::string file_name_; }; class SignCertificateQuery : public Query { @@ -1016,9 +1016,8 @@ class SignCertificateQuery : public Query { void receive_pubkey(td::BufferSlice R); void receive_signature(td::BufferSlice R); - private: - void save_certificate(); + void save_certificate(); td::Bits256 overlay_; td::Bits256 id_; @@ -1077,7 +1076,6 @@ class SignShardOverlayCertificateQuery : public Query { } private: - ton::ShardIdFull shard_; td::int32 expire_at_; ton::PublicKeyHash key_; @@ -1085,7 +1083,6 @@ class SignShardOverlayCertificateQuery : public Query { std::string out_file_; }; - class ImportShardOverlayCertificateQuery : public Query { public: ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1106,7 +1103,6 @@ class ImportShardOverlayCertificateQuery : public Query { } private: - ton::ShardIdFull shard_; ton::PublicKeyHash key_; std::string in_file_; @@ -1442,4 +1438,372 @@ class DelShardQuery : public Query { private: ton::ShardIdFull shard_; -}; \ No newline at end of file +}; + +class AddCollatorQuery : public Query { + public: + AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "add-collator"; + } + static std::string get_help() { + return "add-collator \tadd collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + ton::ShardIdFull shard_; +}; + +class DelCollatorQuery : public Query { + public: + DelCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "del-collator"; + } + static std::string get_help() { + return "del-collator \tremove collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + ton::ShardIdFull shard_; +}; + +class CollatorNodeAddWhitelistedValidatorQuery : public Query { + public: + CollatorNodeAddWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collator-whitelist-add"; + } + static std::string get_help() { + return "collator-whitelist-add \tadd validator adnl id to collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeDelWhitelistedValidatorQuery : public Query { + public: + CollatorNodeDelWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collator-whitelist-del"; + } + static std::string get_help() { + return "collator-whitelist-del \tremove validator adnl id from collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeEnableWhitelistQuery : public Query { + public: + CollatorNodeEnableWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collator-whitelist-enable"; + } + static std::string get_help() { + return "collator-whitelist-enable \tenable or disable collator node whiltelist (value is 0 or 1)"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + +class CollatorNodeShowWhitelistQuery : public Query { + public: + CollatorNodeShowWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collator-whitelist-show"; + } + static std::string get_help() { + return "collator-whitelist-show\tshow collator node whitelist"; + } + std::string name() const override { + return get_name(); + } +}; + +class SetCollatorsListQuery : public Query { + public: + SetCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-collators-list"; + } + static std::string get_help() { + return "set-collators-list \tset list of collators from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearCollatorsListQuery : public Query { + public: + ClearCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clear-collators-list"; + } + static std::string get_help() { + return "clear-collators-list\tclear list of collators"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowCollatorsListQuery : public Query { + public: + ShowCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "show-collators-list"; + } + static std::string get_help() { + return "show-collators-list\tshow list of collators"; + } + std::string name() const override { + return get_name(); + } +}; + +class GetCollationManagerStatsQuery : public Query { + public: + GetCollationManagerStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collation-manager-stats"; + } + static std::string get_help() { + return "collation-manager-stats\tshow stats of collation manager"; + } + std::string name() const override { + return get_name(); + } +}; + +class SignOverlayMemberCertificateQuery : public Query { + public: + SignOverlayMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "sign-overlay-member-certificate"; + } + static std::string get_help() { + return "sign-overlay-member-certificate \tsign overlay member " + "certificate for (hex) with (hex) in slot , valid until , " + "save to "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 key_hash_; + td::Bits256 adnl_id_; + int slot_; + ton::UnixTime expire_at_; + std::string file_name_; +}; + +class ImportFastSyncMemberCertificateQuery : public Query { + public: + ImportFastSyncMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "import-fast-sync-member-certificate"; + } + static std::string get_help() { + return "import-fast-sync-membe-rcertificate \timport member certificate for fast sync overlay " + "for (hex) from "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + std::string file_name_; +}; + +class AddFastSyncOverlayClientQuery : public Query { + public: + AddFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "add-fast-sync-overlay-client"; + } + static std::string get_help() { + return "add-fast-sync-overlay-client \tstarts issuing member certificates to (hex) on " + "slot (int)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + td::int32 slot_; +}; + +class DelFastSyncOverlayClientQuery : public Query { + public: + DelFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "del-fast-sync-overlay-client"; + } + static std::string get_help() { + return "del-fast-sync-overlay-client \tstops issuing member certificates to (hex)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; +}; + +class SetShardBlockVerifierConfigQuery : public Query { + public: + SetShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-shard-block-verifier-config"; + } + static std::string get_help() { + return "set-shard-block-verifier-config \tset config for shard block verifier from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearShardBlockVerifierConfigQuery : public Query { + public: + ClearShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clear-shard-block-verifier-config"; + } + static std::string get_help() { + return "clear-shard-block-verifier-config \treset config for shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowShardBlockVerifierConfigQuery : public Query { + public: + ShowShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "show-shard-block-verifier-config"; + } + static std::string get_help() { + return "show-shard-block-verifier-config\tshow config of shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 234cd6a5f..a747a648f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -156,6 +156,23 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 73949d808..693becdc8 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2ea04e183..cdeb5d917 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,9 +27,17 @@ */ #include "validator-engine.hpp" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "errorcode.h" +#include "keys/keys.hpp" #include "overlay-manager.h" +#include "overlays.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "tl-utils/tl-utils.hpp" #include "tl/TlObject.h" #include "ton/ton-types.h" @@ -59,6 +67,8 @@ #include "memprof/memprof.h" #include "dht/dht.hpp" +#include +#include #if TD_DARWIN || TD_LINUX #include @@ -75,6 +85,7 @@ #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" #include "interfaces/validator-manager.h" +#include "tl-utils/lite-utils.hpp" #if TON_USE_JEMALLOC #include @@ -146,6 +157,11 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); } } + for (auto &col : config.collators_) { + auto id = ton::adnl::AdnlNodeIdShort{col->adnl_id_}; + ton::ShardIdFull shard = ton::create_shard_id(col->shard_); + config_add_collator(id, shard).ensure(); + } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); for (auto &s : config.fullnodeslaves_) { @@ -163,6 +179,23 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } if (config.extraconfig_) { state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + for (auto &f : config.extraconfig_->fast_sync_member_certificates_) { + ton::adnl::AdnlNodeIdShort adnl_id{f->adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{f->certificate_.get()}; + if (!certificate.empty() && !certificate.is_expired()) { + fast_sync_member_certificates.emplace_back(adnl_id, std::move(certificate)); + } + } + if (config.extraconfig_->collator_node_whitelist_) { + collator_node_whiltelist_enabled = config.extraconfig_->collator_node_whitelist_->enabled_; + for (const auto& id : config.extraconfig_->collator_node_whitelist_->adnl_ids_) { + collator_node_whitelist.emplace(id); + } + } + for (auto &client : config.extraconfig_->fast_sync_overlay_clients_) { + auto key = ton::adnl::AdnlNodeIdShort{client->adnl_id_}; + fast_sync_overlay_clients.emplace_back(std::move(key), client->slot_); + } } else { state_serializer_enabled = true; } @@ -229,6 +262,13 @@ ton::tl_object_ptr Config::tl() const { val_vec.push_back(ton::create_tl_object( val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } + std::vector> col_vec; + for (auto &[col, shards] : collators) { + for (auto &shard : shards) { + col_vec.push_back( + ton::create_tl_object(col.bits256_value(), ton::create_tl_shard_id(shard))); + } + } std::vector> full_node_slaves_vec; for (auto &x : full_node_slaves) { @@ -246,10 +286,32 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr collator_node_whitelist_obj = {}; + if (collator_node_whiltelist_enabled || !collator_node_whitelist.empty()) { + collator_node_whitelist_obj = ton::create_tl_object(); + collator_node_whitelist_obj->enabled_ = collator_node_whiltelist_enabled; + for (const auto& id : collator_node_whitelist) { + collator_node_whitelist_obj->adnl_ids_.push_back(id.bits256_value()); + } + } + ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj || + !fast_sync_overlay_clients.empty()) { // Non-default values - extra_config_obj = ton::create_tl_object(state_serializer_enabled); + extra_config_obj = ton::create_tl_object(); + extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; + for (const auto &[adnl_id, certificate] : fast_sync_member_certificates) { + extra_config_obj->fast_sync_member_certificates_.push_back( + ton::create_tl_object(adnl_id.bits256_value(), + certificate.tl())); + } + extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); + for (const auto &client : fast_sync_overlay_clients) { + extra_config_obj->fast_sync_overlay_clients_.push_back( + ton::create_tl_object(client.id.bits256_value(), + client.slot)); + } } std::vector> liteserver_vec; @@ -278,7 +340,7 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); @@ -425,6 +487,31 @@ td::Result Config::config_add_validator_adnl_id(ton::PublicKeyHash perm_ke } } +td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + auto& shards = collators[addr]; + if (std::find(shards.begin(), shards.end(), shard) != shards.end()) { + return false; + } + shards.push_back(shard); + return true; +} + +td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + auto& shards = collators[addr]; + auto it = std::find(shards.begin(), shards.end(), shard); + if (it == shards.end()) { + return false; + } + shards.erase(it); + return true; +} + td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { if (full_node == id) { return false; @@ -1276,6 +1363,9 @@ void ValidatorEngine::set_local_config(std::string str) { void ValidatorEngine::set_global_config(std::string str) { global_config_ = str; } +void ValidatorEngine::set_tunnel_config(std::string str) { + tunnel_config_ = str; +} void ValidatorEngine::set_db_root(std::string db_root) { db_root_ = db_root; } @@ -1285,10 +1375,12 @@ void ValidatorEngine::schedule_shutdown(double at) { LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; } else { LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; - ton::delay_action([]() { - LOG(WARNING) << "Shutting down as scheduled"; - std::_Exit(0); - }, ts); + ton::delay_action( + []() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, + ts); } } void ValidatorEngine::start_up() { @@ -1321,12 +1413,14 @@ void ValidatorEngine::alarm() { auto cur_t = config->get_validator_set_start_stop(0); CHECK(cur_t.first > 0); - auto val_set = state_->get_total_validator_set(0); - auto e = val_set->export_vector(); std::set to_del; for (auto &val : config_.validators) { bool is_validator = false; - if (val_set->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + if (validator_set_next_.not_null() && + validator_set_next_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + is_validator = true; + } + if (validator_set_.not_null() && validator_set_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { is_validator = true; } if (!is_validator && val.second.election_date < cur_t.first && cur_t.first + 600 < state_->get_unix_time()) { @@ -1347,9 +1441,41 @@ void ValidatorEngine::alarm() { need_write = true; } + { + std::set fs_to_del; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.second.is_expired()) { + fs_to_del.insert(x.first); + continue; + } + auto issued_by = x.second.issued_by().compute_short_id().bits256_value(); + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + continue; + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + continue; + } + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + continue; + } + fs_to_del.insert(x.first); + } + if (!fs_to_del.empty()) { + need_write = true; + std::erase_if(config_.fast_sync_member_certificates, + [&](const std::pair &e) { + return !fs_to_del.contains(e.first); + }); + } + } + if (need_write) { write_config([](td::Unit) {}); } + if (issue_fast_sync_overlay_certificates_at_.is_in_past()) { + issue_fast_sync_overlay_certificates_at_ = td::Timestamp::in(60.0); + issue_fast_sync_overlay_certificates(); + } } for (auto &x : config_.gc) { if (running_gc_.count(x) == 0) { @@ -1376,6 +1502,16 @@ void ValidatorEngine::deleted_key(ton::PublicKeyHash x) { } } +void ValidatorEngine::got_state(td::Ref state) { + if (state_.not_null() && state_->get_block_id() == state->get_block_id()) { + return; + } + state_ = std::move(state); + validator_set_ = state_->get_total_validator_set(0); + validator_set_next_ = state_->get_total_validator_set(1); + validator_set_prev_ = state_->get_total_validator_set(-1); +} + td::Status ValidatorEngine::load_global_config() { TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -1414,6 +1550,9 @@ td::Status ValidatorEngine::load_global_config() { if (zero_state.root_hash.is_zero() || zero_state.file_hash.is_zero()) { return td::Status::Error(ton::ErrorCode::error, "[validator] section contains incomplete [zero_state]"); } + if (celldb_in_memory_ && celldb_v2_) { + return td::Status::Error(ton::ErrorCode::error, "at most one of --celldb-in-memory --celldb-v2 could be used"); + } ton::BlockIdExt init_block; if (!conf.validator_->init_block_) { @@ -1466,6 +1605,8 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_celldb_v2(celldb_v2_); + validator_options_.write().set_celldb_disable_bloom_filter(celldb_disable_bloom_filter_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -1484,6 +1625,8 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_slow_) { validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); } + validator_options_.write().set_permanent_celldb(permanent_celldb_); + validator_options_.write().set_initial_sync_disabled(skip_key_sync_); std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1506,21 +1649,38 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); + for (auto& id : config_.collator_node_whitelist) { + validator_options_.write().set_collator_node_whitelisted_validator(id, true); + } + validator_options_.write().set_collator_node_whitelist_enabled(config_.collator_node_whiltelist_enabled); + return td::Status::OK(); } void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { - validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + validator_options_.write().set_shard_check_function( + [sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, ton::BlockSeqno mc_seqno) -> bool { + return shard.is_masterchain() || !sync_shards_upto || mc_seqno <= sync_shards_upto.value(); + }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& s : config_.shards_to_monitor) { + for (const auto &[_, collator_shards] : config_.collators) { + for (const auto &shard : collator_shards) { + shards.push_back(shard); + } + } + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + [shards = std::move(shards), sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, + ton::BlockSeqno mc_seqno) -> bool { + if (!shard.is_masterchain() && sync_shards_upto && mc_seqno > sync_shards_upto.value()) { + return false; + } for (auto s : shards) { if (shard_intersects(shard, s)) { return true; @@ -1531,6 +1691,66 @@ void ValidatorEngine::set_shard_check_function() { } } +void ValidatorEngine::load_collators_list() { + collators_list_ = {}; + auto data_R = td::read_file(collators_list_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + collators_list_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*collators_list_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << S; + collators_list_ = {}; + return; + } + td::Ref list{true}; + S = list.write().unpack(*collators_list_); + if (S.is_ok()) { + validator_options_.write().set_collators_list(std::move(list)); + } else { + LOG(ERROR) << "Invalid collators list: " << S.move_as_error(); + collators_list_ = {}; + } +} + +void ValidatorEngine::load_shard_block_verifier_config() { + shard_block_verifier_config_ = {}; + auto data_R = td::read_file(shard_block_verifier_config_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + shard_block_verifier_config_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*shard_block_verifier_config_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << S; + shard_block_verifier_config_ = {}; + return; + } + td::Ref config{true}; + S = config.write().unpack(*shard_block_verifier_config_); + if (S.is_ok()) { + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + } else { + LOG(ERROR) << "Invalid shard block verifier config: " << S.move_as_error(); + shard_block_verifier_config_ = {}; + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1840,6 +2060,8 @@ void ValidatorEngine::got_key(ton::PublicKey key) { void ValidatorEngine::start() { set_shard_check_function(); + load_collators_list(); + load_shard_block_verifier_config(); read_config_ = true; start_adnl(); } @@ -1849,6 +2071,59 @@ void ValidatorEngine::start_adnl() { adnl_ = ton::adnl::Adnl::create(db_root_, keyring_.get()); td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_network_manager, adnl_network_manager_.get()); + if (!tunnel_config_.empty()) { + auto on_tunnel_ready = td::PromiseCreator::lambda([SelfId = actor_id(this), this](td::Result R) { + R.ensure(); + auto addr = R.move_as_ok(); + + LOG(INFO) << "Tunnel ready, addr: " << addr; + + add_addr(Config::Addr{}, Config::AddrCats{ + .in_addr = addr, + .is_tunnel = true, + .cats = {0, 1, 2, 3}, + }); + + for (auto &adnl : config_.adnl_ids) { + add_adnl(adnl.first, adnl.second); + } + + td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, std::move(adnl_static_nodes_)); + td::actor::send_closure(SelfId, &ValidatorEngine::started_adnl); + }); + + class Handler : public ton::adnl::AdnlNetworkManager::TunnelEventsHandler { + public: + Handler(td::actor::Scheduler *scheduler, const td::actor::ActorId &actor) + : scheduler_(scheduler), validator_engine_actor_(actor) { + } + + private: + td::actor::Scheduler* scheduler_; + td::actor::ActorId validator_engine_actor_; + + void on_in_addr_update(td::IPAddress ip) override { + LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; + + scheduler_->run_in_context_external([&] { + td::actor::send_closure(validator_engine_actor_, &ValidatorEngine::reinit_tunnel, ip); + }); + } + }; + + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this->scheduler_, actor_id(this))); + + ton::adnl::AdnlCategoryMask cat_mask; + for (int i = 0; i <= 3; i++) { + cat_mask[i] = true; + } + + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_tunnel, global_config_, tunnel_config_, + std::move(cat_mask), 0, std::move(on_tunnel_ready), scheduler_); + + return; + } + for (auto &addr : config_.addrs) { add_addr(addr.first, addr.second); } @@ -1860,6 +2135,19 @@ void ValidatorEngine::start_adnl() { started_adnl(); } +void ValidatorEngine::reinit_tunnel(td::IPAddress ip) { + this->addr_lists_.clear(); + this->add_addr(Config::Addr{}, Config::AddrCats{ + .in_addr = ip, + .is_tunnel = true, + .cats = {0, 1, 2, 3}, + }); + + for (auto &adnl : this->config_.adnl_ids) { + this->add_adnl(adnl.first, adnl.second); + } +} + void ValidatorEngine::add_addr(const Config::Addr &addr, const Config::AddrCats &cats) { ton::adnl::AdnlCategoryMask cat_mask; for (auto cat : cats.cats) { @@ -1868,13 +2156,16 @@ void ValidatorEngine::add_addr(const Config::Addr &addr, const Config::AddrCats for (auto cat : cats.priority_cats) { cat_mask[cat] = true; } - if (!cats.proxy) { - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_self_addr, addr.addr, - std::move(cat_mask), cats.cats.size() ? 0 : 1); - } else { - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_proxy_addr, cats.in_addr, - static_cast(addr.addr.get_port()), cats.proxy, std::move(cat_mask), - cats.cats.size() ? 0 : 1); + + if (!cats.is_tunnel) { + if (!cats.proxy) { + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_self_addr, addr.addr, + std::move(cat_mask), cats.cats.size() ? 0 : 1); + } else { + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::add_proxy_addr, cats.in_addr, + static_cast(addr.addr.get_port()), cats.proxy, std::move(cat_mask), + cats.cats.size() ? 0 : 1); + } } td::uint32 ts = static_cast(td::Clocks::system()); @@ -1905,7 +2196,12 @@ void ValidatorEngine::started_adnl() { } void ValidatorEngine::add_dht(ton::PublicKeyHash id) { - auto D = ton::dht::Dht::create(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); + auto creator = ton::dht::Dht::create; + if (!tunnel_config_.empty()) { + creator = ton::dht::Dht::create_client; + } + + auto D = creator(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); D.ensure(); dht_nodes_[id] = D.move_as_ok(); @@ -1963,7 +2259,7 @@ void ValidatorEngine::start_validator() { load_collator_options(); validator_manager_ = ton::validator::ValidatorManagerFactory::create( - validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); + validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), overlay_manager_.get()); for (auto &v : config_.validators) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_permanent_key, v.first, @@ -1975,6 +2271,18 @@ void ValidatorEngine::start_validator() { } } + if (shard_block_retainer_adnl_id_fullnode_) { + shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; + } + if (!shard_block_retainer_adnl_id_.is_zero()) { + if (config_.adnl_ids.contains(shard_block_retainer_adnl_id_.pubkey_hash())) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_shard_block_retainer, + shard_block_retainer_adnl_id_); + } else { + LOG(ERROR) << "Cannot start shard block retainer: no adnl id " << shard_block_retainer_adnl_id_; + } + } + started_validator(); } @@ -1983,7 +2291,8 @@ void ValidatorEngine::started_validator() { } void ValidatorEngine::start_full_node() { - if (!config_.full_node.is_zero() || config_.full_node_slaves.size() > 0) { + if (!config_.full_node.is_zero() || !config_.full_node_slaves.empty()) { + full_node_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; auto short_id = pk.compute_short_id(); td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {}); @@ -2010,7 +2319,7 @@ void ValidatorEngine::start_full_node() { .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_}; full_node_ = ton::validator::fullnode::FullNode::create( - short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, + short_id, full_node_id_, validator_options_->zero_block_id().file_hash, full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); @@ -2018,11 +2327,19 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } - load_custom_overlays_config(); + for (auto &[c, _] : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, c); + } + for (auto &x : config_.fast_sync_member_certificates) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + } if (!validator_telemetry_filename_.empty()) { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::set_validator_telemetry_filename, validator_telemetry_filename_); } + load_custom_overlays_config(); + register_fast_sync_certificate_callback(); } else { started_full_node(); } @@ -2049,6 +2366,20 @@ void ValidatorEngine::start_lite_server() { } void ValidatorEngine::started_lite_server() { + start_collator(); +} + +void ValidatorEngine::start_collator() { + for (auto& [id, shards] : config_.collators) { + for (auto& shard : shards) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); + } + } + + started_collator(); +} + +void ValidatorEngine::started_collator() { start_control_interface(); } @@ -2248,9 +2579,17 @@ void ValidatorEngine::try_add_full_node_adnl_addr(ton::PublicKeyHash id, td::Pro return; } - if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, - ton::adnl::AdnlNodeIdShort{id}, [](td::Unit) {}); + if (!full_node_.empty() && id != full_node_id_.pubkey_hash()) { + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID)); + full_node_id_ = ton::adnl::AdnlNodeIdShort{id}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, full_node_id_, + [](td::Unit) {}); + register_fast_sync_certificate_callback(); } write_config(std::move(promise)); @@ -2510,9 +2849,225 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector validator_engine) + : validator_engine_(std::move(validator_engine)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + auto R = ton::fetch_tl_object( + std::move(data), true); + if (R.is_error()) { + return; + } + auto res = R.move_as_ok(); + auto cert = ton::overlay::OverlayMemberCertificate(res->certificate_.get()); + if (cert.empty()) { + return; + } + LOG(DEBUG) << "Received tonNode.newFastSyncMemberCertificate from " << src; + td::actor::send_closure(validator_engine_, &ValidatorEngine::try_import_fast_sync_member_certificate, dst, + std::move(cert), td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "failed to import overlay member certificate: " << R.move_as_error(); + } + })); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + auto R = + ton::fetch_tl_object(std::move(data), true); + if (R.is_error()) { + return; + } + auto q = R.move_as_ok(); + td::actor::send_closure( + validator_engine_, &ValidatorEngine::process_fast_sync_overlay_certificate_request, + ton::PublicKeyHash{q->sign_by_}, ton::adnl::AdnlNodeIdShort{q->adnl_id_}, 0, q->slot_, + (td::int32)td::Clocks::system() + 3600, + td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); + })); + } + + private: + td::actor::ActorId validator_engine_; + }; + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID), + std::make_unique(actor_id(this))); +} + +void ValidatorEngine::try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise) { + if (!started_ || state_.is_null()) { + return promise.set_error(td::Status::Error("not started")); + } + if (certificate.slot() < 0 || + certificate.slot() >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + return promise.set_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)")); + } + if (certificate.expire_at() < td::Clocks::system() + 60) { + return promise.set_error(td::Status::Error("certificate expires too soon")); + } + TRY_STATUS_PROMISE_PREFIX(promise, certificate.check_signature(id), "invalid certificate: "); + + auto cert_score = [this](ton::overlay::OverlayMemberCertificate &cert) -> td::int64 { + auto issued_by = cert.issued_by().compute_short_id().bits256_value(); + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + return cert.expire_at() + (0ll << 32); + } + return -1; + }; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.first == id) { + // fast check + if (x.second.issued_by() == certificate.issued_by()) { + if (x.second.expire_at() < certificate.expire_at()) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate from the same issuer exists with bigger ttl"; + promise.set_value(td::Unit()); + return; + } + auto new_score = cert_score(certificate); + auto old_score = cert_score(x.second); + if (new_score > old_score) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate with better score exists"; + promise.set_value(td::Unit()); + return; + } + } + + auto new_score = cert_score(certificate); + if (new_score < 0) { + LOG(DEBUG) << "Not importing certificate: issuer is not a validator"; + promise.set_value(td::Unit()); + return; + } + + auto &x = config_.fast_sync_member_certificates.emplace_back(std::move(id), std::move(certificate)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, x.first, + x.second); + write_config(std::move(promise)); +} + +void ValidatorEngine::issue_fast_sync_overlay_certificates() { + if (state_.is_null() || config_.fast_sync_overlay_clients.empty() || full_node_id_.is_zero()) { + return; + } + auto issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return; + } + auto src = full_node_id_; + td::int32 expire_at = static_cast(td::Clocks::system()) + 3600; + for (auto &client : config_.fast_sync_overlay_clients) { + issue_fast_sync_overlay_certificate( + issue_by, client.id, 0, client.slot, expire_at, + [src, dst = client.id, adnl = adnl_.get()](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "cannot issue fast sync overlay certificate for " << dst << ": " << R.move_as_error(); + return; + } + auto cert = R.move_as_ok(); + LOG(INFO) << "Sending fast sync overlay certificate issued by " << cert.issued_by().compute_short_id() + << " to " << dst << " slot " << cert.slot(); + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, + ton::create_serialize_tl_object( + dst.bits256_value(), cert.tl())); + }); + } +} + +void ValidatorEngine::issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, + ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, + td::int32 slot, td::int32 expire_at, + td::Promise promise) { + if (issue_by.is_zero()) { + issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return promise.set_error(td::Status::Error(ton::ErrorCode::notready, "cannot find a local validator")); + } + } + if (expire_at < td::Clocks::system() + 10) { + return promise.set_error(td::Status::Error(ton::ErrorCode::error, "expire at is in too near future")); + } + ton::overlay::OverlayMemberCertificate cert(ton::PublicKey(), flags, slot, expire_at, td::BufferSlice()); + auto to_sign = cert.to_sign_data(issue_to); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, issue_by, std::move(to_sign), + [slot, flags, expire_at, promise = std::move(promise)]( + td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), "failed to sign certificate: "); + ton::overlay::OverlayMemberCertificate cert(std::move(res.second), flags, slot, expire_at, + std::move(res.first)); + promise.set_value(std::move(cert)); + }); +} + +void ValidatorEngine::process_fast_sync_overlay_certificate_request( + ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, td::int32 slot, + td::int32 expire_at, td::Promise promise) { + for (auto &client : config_.fast_sync_overlay_clients) { + if (client.id == issue_to && (slot < 0 || slot == client.slot)) { + return issue_fast_sync_overlay_certificate(std::move(issue_by), std::move(issue_to), flags, client.slot, + expire_at, std::move(promise)); + } + } + promise.set_error(td::Status::Error(ton::ErrorCode::error, "cannot issue certificate to unknown adnl id")); +} + +ton::PublicKeyHash ValidatorEngine::find_local_validator_for_cert_issuing() { + if (state_.is_null()) { + return ton::PublicKeyHash{}; + } + for (auto& val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { + if (val_set.is_null()) { + continue; + } + for (auto &[val_id, _]: config_.validators) { + if (val_set->is_validator(ton::NodeIdShort{val_id.bits256_value()})) { + return val_id; + } + } + } + return ton::PublicKeyHash::zero(); +} + void ValidatorEngine::load_custom_overlays_config() { - custom_overlays_config_ = - ton::create_tl_object(); + custom_overlays_config_ = ton::create_tl_object(); auto data_R = td::read_file(custom_overlays_config_file()); if (data_R.is_error()) { return; @@ -2565,7 +3120,7 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi static td::Result> parse_collator_options(td::MutableSlice json_str) { td::Ref ref{true}; - ton::validator::CollatorOptions& opts = ref.write(); + ton::validator::CollatorOptions &opts = ref.write(); // Set default values (from_json leaves missing fields as is) ton::ton_api::engine_validator_collatorOptions f; @@ -2577,6 +3132,8 @@ static td::Result> parse_collator_optio f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator; f.dispatch_phase_3_max_per_initiator_ = opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1; + f.force_full_collated_data_ = false; + f.ignore_collated_data_limits_ = false; TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: "); TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: "); @@ -2616,6 +3173,8 @@ static td::Result> parse_collator_optio TRY_RESULT(addr, block::StdAddress::parse(s)); opts.prioritylist.emplace(addr.workchain, addr.addr); } + opts.force_full_collated_data = f.force_full_collated_data_; + opts.ignore_collated_data_limits = f.ignore_collated_data_limits_; return ref; } @@ -3568,7 +4127,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } //TODO force Overlays::update_certificate to return result @@ -3583,17 +4142,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer }); */ td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, - ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, - ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.signed_key_->key_hash_}, - r.move_as_ok()); - promise.set_value( - ton::serialize_tl_object(ton::create_tl_object(), true) - ); + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3608,7 +4165,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -3620,12 +4177,13 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha } }); ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3647,11 +4205,11 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard promise.set_value(R.move_as_ok()); } }); - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, + std::move(P)); } - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3719,42 +4277,46 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi return; } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { - const std::vector times{60, 300, 3600}; - double now = td::Time::now(); - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - auto r = R.move_as_ok(); - std::vector> by_name; - for (const auto &stats : r) { - if (stats.name == query.name_ || query.name_.empty()) { - std::vector> by_time; - for (const auto &t : times) { - double min = std::numeric_limits::lowest(); - double max = std::numeric_limits::max(); - double sum = 0; - int cnt = 0; - for (const auto &stat : stats.stats) { - double time = stat.first; - double duration = stat.second; - if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); - sum += duration; - ++cnt; - } - } - by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), query = std::move(query)]( + td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; } - by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); } + by_time.push_back(ton::create_tl_object( + t, min, sum / static_cast(cnt), max)); } - promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + by_name.push_back(ton::create_tl_object( + stats.name, std::move(by_time))); } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); + } + promise.set_value( + ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, + std::move(P)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, @@ -3877,9 +4439,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMes }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3948,9 +4509,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4019,10 +4579,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat promise.set_value(ton::create_serialize_tl_object()); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; } @@ -4030,11 +4590,97 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollat promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); return; } - auto r_data = td::read_file(collator_options_file()); - if (r_data.is_error()) { - promise.set_value(ton::create_serialize_tl_object("{}")); + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + if (query.add_) { + if (!config_.collator_node_whitelist.insert(adnl_id).second) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } } else { - promise.set_value( + if (config_.collator_node_whitelist.erase(adnl_id) == 0) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + } + + validator_options_.write().set_collator_node_whitelisted_validator(adnl_id, query.add_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (config_.collator_node_whiltelist_enabled == query.enabled_) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + config_.collator_node_whiltelist_enabled = query.enabled_; + validator_options_.write().set_collator_node_whitelist_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::tl_object_ptr result = {}; + result = ton::create_tl_object(); + result->enabled_ = config_.collator_node_whiltelist_enabled; + for (const auto &id : config_.collator_node_whitelist) { + result->adnl_ids_.push_back(id.bits256_value()); + } + promise.set_value(ton::serialize_tl_object(result, true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r_data = td::read_file(collator_options_file()); + if (r_data.is_error()) { + promise.set_value(ton::create_serialize_tl_object("{}")); + } else { + promise.set_value( ton::create_serialize_tl_object(r_data.ok().as_slice().str())); } } @@ -4088,7 +4734,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4124,9 +4771,386 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::Ref list{true}; + auto S = list.write().unpack(*query.list_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid collators list: "))); + return; + } + auto s = td::json_encode(td::ToJson(*query.list_), true); + S = td::write_file(collators_list_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + collators_list_ = std::move(query.list_); + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto S = td::unlink(collators_list_file()); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + td::Ref list{true, ton::validator::CollatorsList::default_list()}; + collators_list_ = {}; + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (collators_list_) { + promise.set_value(ton::serialize_tl_object(collators_list_, true)); + } else { + promise.set_value(create_control_query_error(td::Status::Error("collators list is empty"))); + } +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_collation_manager_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_add_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); + } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, id); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such collator"))); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, id, shard); + } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, id); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::PublicKeyHash public_key_hash{query.sign_by_}; + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + int slot = query.slot_; + int expire_at = query.expire_at_; + + issue_fast_sync_overlay_certificate( + public_key_hash, adnl_id, 0, slot, expire_at, + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{query.certificate_.get()}; + if (certificate.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is empty"))); + return; + } + td::Status S = certificate.check_signature(adnl_id); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + try_import_fast_sync_member_certificate( + std::move(adnl_id), std::move(certificate), [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + td::int32 slot = query.slot_; + if (slot < 0 || slot >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + promise.set_value(create_control_query_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)"))); + return; + } + + bool found = false; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.slot == slot) { + if (c.id == adnl_id) { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + issue_fast_sync_overlay_certificates(); + } else { + promise.set_value(create_control_query_error(td::Status::Error("duplicate slot"))); + } + return; + } + if (c.id == adnl_id) { + found = true; + c.slot = slot; + } + } + if (!found) { + config_.fast_sync_overlay_clients.emplace_back(adnl_id, slot); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); + issue_fast_sync_overlay_certificates(); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.id == adnl_id) { + std::swap(c, config_.fast_sync_overlay_clients.back()); + config_.fast_sync_overlay_clients.pop_back(); + break; + } + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::Ref config{true}; + auto S = config.write().unpack(*query.config_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid config: "))); + return; + } + auto s = td::json_encode(td::ToJson(*query.config_), true); + S = td::write_file(shard_block_verifier_config_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + shard_block_verifier_config_ = std::move(query.config_); + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (shard_block_verifier_config_) { + promise.set_value(ton::serialize_tl_object(shard_block_verifier_config_, true)); + } else { + promise.set_value(create_control_query_error(td::Status::Error("shard block verifier config is empty"))); + } } void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, @@ -4198,9 +5222,8 @@ void ValidatorEngine::get_current_validator_perm_key(td::Promiseget_total_validator_set(0); - CHECK(val_set.not_null()); - auto vec = val_set->export_vector(); + CHECK(validator_set_.not_null()); + auto vec = validator_set_->export_vector(); for (size_t idx = 0; idx < vec.size(); idx++) { auto &el = vec[idx]; ton::PublicKey pub{ton::pubkeys::Ed25519{el.key.as_bits256()}}; @@ -4311,6 +5334,10 @@ int main(int argc, char *argv[]) { acts.push_back( [&x, fname = fname.str()]() { td::actor::send_closure(x, &ValidatorEngine::set_local_config, fname); }); }); + p.add_option('\0', "tunnel-config", "file to read tunnel config", [&](td::Slice fname) { + acts.push_back( + [&x, fname = fname.str()]() { td::actor::send_closure(x, &ValidatorEngine::set_tunnel_config, fname); }); + }); p.add_checked_option('I', "ip", "ip:port of instance", [&](td::Slice arg) { td::IPAddress addr; TRY_STATUS(addr.init_host_port(arg.str())); @@ -4355,7 +5382,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", + p.add_checked_option('b', "block-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4365,7 +5392,9 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", + 'A', "archive-ttl", + "ttl for archived blocks (in seconds) default=7*86400. Note: archived blocks are gc'd after state-ttl + " + "archive-ttl seconds", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4375,7 +5404,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + 'K', "key-proof-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4526,6 +5555,18 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option( + '\0', "celldb-v2", + "use new version off celldb", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); + p.add_option( + '\0', "celldb-disable-bloom-filter", + "disable using bloom filter in CellDb. Enabled bloom filter reduces read latency, but increases memory usage", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_disable_bloom_filter, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { @@ -4555,7 +5596,7 @@ int main(int argc, char *argv[]) { }); p.add_option( '\0', "collect-validator-telemetry", - "store validator telemetry from private block overlay to a given file (json format)", + "store validator telemetry from fast sync overlay to a given file (json format)", [&](td::Slice s) { acts.push_back( [&x, s = s.str()]() { @@ -4569,7 +5610,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-catchain", - "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in catchain overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4581,7 +5622,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-public", - "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in public shard overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4593,7 +5634,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-private", - "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in private block overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4603,6 +5644,40 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_option( + '\0', "permanent-celldb", + "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " + "cannot be disabled)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); + p.add_option('\0', "skip-key-sync", + "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); + }); + p.add_checked_option( + '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); + return td::Status::OK(); + }); + p.add_checked_option('\0', "shard-block-retainer", + "adnl id for shard block retainer (hex), or \"fullnode\" for full node id", + [&](td::Slice arg) -> td::Status { + if (arg == "fullnode") { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id_fullnode); + }); + } else { + td::Bits256 v; + if (v.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, + ton::adnl::AdnlNodeIdShort{v}); + }); + } + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); @@ -4617,7 +5692,7 @@ int main(int argc, char *argv[]) { scheduler.run_in_context([&] { vm::init_vm().ensure(); - x = td::actor::create_actor("validator-engine"); + x = td::actor::create_actor("validator-engine", &scheduler); for (auto &act : acts) { act(); } diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0dc91f13..1ea4a372f 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -27,11 +27,14 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "overlays.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "validator/manager.h" #include "validator/validator.h" #include "validator/full-node.h" @@ -57,6 +60,7 @@ struct Config { }; struct AddrCats { td::IPAddress in_addr; + bool is_tunnel; std::shared_ptr proxy; std::set cats; std::set priority_cats; @@ -75,6 +79,13 @@ struct Config { ton::PublicKey key; td::IPAddress addr; }; + struct FastSyncOverlayClient { + FastSyncOverlayClient() = default; + FastSyncOverlayClient(ton::adnl::AdnlNodeIdShort id, td::int32 slot) : id(id), slot(slot) { + } + ton::adnl::AdnlNodeIdShort id; + td::int32 slot; + }; std::map keys_refcnt; td::uint16 out_port; @@ -82,6 +93,9 @@ struct Config { std::map adnl_ids; std::set dht_ids; std::map validators; + std::map> collators; + bool collator_node_whiltelist_enabled = false; + std::set collator_node_whitelist; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); std::vector full_node_slaves; std::map full_node_masters; @@ -90,8 +104,11 @@ struct Config { std::map controls; std::set gc; std::vector shards_to_monitor; + std::vector fast_sync_overlay_clients; bool state_serializer_enabled = true; + std::vector> + fast_sync_member_certificates; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -109,6 +126,8 @@ struct Config { ton::UnixTime expire_at); td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); + td::Result config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); + td::Result config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -140,6 +159,8 @@ struct Config { class ValidatorEngine : public td::actor::Actor { private: + td::actor::Scheduler* scheduler_; + td::actor::ActorOwn keyring_; td::actor::ActorOwn adnl_network_manager_; td::actor::ActorOwn adnl_; @@ -151,11 +172,13 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn validator_manager_; td::actor::ActorOwn full_node_client_; td::actor::ActorOwn full_node_; + ton::adnl::AdnlNodeIdShort full_node_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::map> full_node_masters_; td::actor::ActorOwn control_ext_server_; std::string local_config_ = ""; std::string global_config_ = "ton-global.config"; + std::string tunnel_config_ = ""; std::string config_file_; std::string temp_config_file() const { return config_file_ + ".tmp"; @@ -173,21 +196,24 @@ class ValidatorEngine : public td::actor::Actor { td::Ref validator_options_; Config config_; ton::tl_object_ptr custom_overlays_config_; + ton::tl_object_ptr collators_list_; + ton::tl_object_ptr shard_block_verifier_config_; std::set running_gc_; std::map keys_; td::Ref state_; + td::Ref validator_set_, validator_set_prev_, validator_set_next_; + td::Timestamp issue_fast_sync_overlay_certificates_at_ = td::Timestamp::now(); td::Promise get_key_promise(td::MultiPromise::InitGuard &ig); void got_key(ton::PublicKey key); void deleted_key(ton::PublicKeyHash key); - void got_state(td::Ref state) { - state_ = std::move(state); - } + void got_state(td::Ref state); void write_config(td::Promise promise); + void reinit_tunnel(td::IPAddress ip); std::map addr_lists_; std::map prio_addr_lists_; @@ -218,6 +244,8 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -229,9 +257,14 @@ class ValidatorEngine : public td::actor::Actor { bool not_all_shards_ = false; std::vector add_shard_cmds_; bool state_serializer_disabled_flag_ = false; - double broadcast_speed_multiplier_catchain_ = 1.0; - double broadcast_speed_multiplier_public_ = 1.0; - double broadcast_speed_multiplier_private_ = 1.0; + double broadcast_speed_multiplier_catchain_ = 3.33; + double broadcast_speed_multiplier_public_ = 3.33; + double broadcast_speed_multiplier_private_ = 3.33; + bool permanent_celldb_ = false; + bool skip_key_sync_ = false; + td::optional sync_shards_upto_; + ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); + bool shard_block_retainer_adnl_id_fullnode_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -249,6 +282,7 @@ class ValidatorEngine : public td::actor::Actor { } void set_local_config(std::string str); void set_global_config(std::string str); + void set_tunnel_config(std::string str); void set_fift_dir(std::string str) { fift_dir_ = str; } @@ -311,6 +345,12 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_in_memory(bool value) { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) { + celldb_v2_ = value; + } + void set_celldb_disable_bloom_filter(bool value) { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } @@ -341,9 +381,24 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_permanent_celldb(bool value) { + permanent_celldb_ = value; + } + void set_skip_key_sync(bool value) { + skip_key_sync_ = value; + } + void set_sync_shards_upto(ton::BlockSeqno seqno) { + sync_shards_upto_ = seqno; + } + void set_shard_block_retainer_adnl_id(ton::adnl::AdnlNodeIdShort id) { + shard_block_retainer_adnl_id_ = id; + } + void set_shard_block_retainer_adnl_id_fullnode() { + shard_block_retainer_adnl_id_fullnode_ = true; + } void start_up() override; - ValidatorEngine() { + explicit ValidatorEngine(td::actor::Scheduler* scheduler): scheduler_(scheduler) { } // load config @@ -352,6 +407,8 @@ class ValidatorEngine : public td::actor::Actor { void load_local_config(td::Promise promise); void load_config(td::Promise promise); void set_shard_check_function(); + void load_collators_list(); + void load_shard_block_verifier_config(); void start(); @@ -379,6 +436,8 @@ class ValidatorEngine : public td::actor::Actor { void add_lite_server(ton::PublicKeyHash id, td::uint16 port); void start_lite_server(); void started_lite_server(); + void start_collator(); + void started_collator(); void add_control_interface(ton::PublicKeyHash id, td::uint16 port); void add_control_process(ton::PublicKeyHash id, td::uint16 port, ton::PublicKeyHash pub, td::int32 permissions); @@ -425,17 +484,37 @@ class ValidatorEngine : public td::actor::Actor { void try_del_proxy(td::uint32 ip, td::int32 port, std::vector cats, std::vector prio_cats, td::Promise promise); + void register_fast_sync_certificate_callback(); + void try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise); + + void issue_fast_sync_overlay_certificates(); + void issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + void process_fast_sync_overlay_certificate_request(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + ton::PublicKeyHash find_local_validator_for_cert_issuing(); + std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } std::string collator_options_file() const { return db_root_ + "/collator-options.json"; } + std::string collators_list_file() const { + return db_root_ + "/collators-list.json"; + } + std::string shard_block_verifier_config_file() const { + return db_root_ + "/shard-block-verifier-config.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); - void add_custom_overlay_to_config( - ton::tl_object_ptr overlay, td::Promise promise); + void add_custom_overlay_to_config(ton::tl_object_ptr overlay, + td::Promise promise); void del_custom_overlay_from_config(std::string name, td::Promise promise); void load_collator_options(); @@ -519,6 +598,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, td::BufferSlice data, @@ -535,10 +618,37 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 6b4e8f687..6d4b00bd6 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -28,4 +26,4 @@ target_include_directories(validatorsession PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain) +target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp2 tl_api dht tdfec overlay catchain) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 4442504ed..9aa376e2a 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -18,6 +18,7 @@ #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" #include "td/utils/lz4.h" +#include "vm/boc-compression.h" #include "validator-session-types.h" namespace ton::validatorsession { @@ -35,17 +36,40 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size) { + int max_decompressed_data_size, + int proto_version) { if (!compression_enabled) { return fetch_tl_object(data, true); } - TRY_RESULT(f, fetch_tl_object(data, true)); - if (f->decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), - std::move(p.second)); + TRY_RESULT(f, fetch_tl_object(data, true)); + td::Result> res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::validatorSession_candidate& c) { + res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); + }, + [&](ton_api::validatorSession_compressedCandidate& c) { + res = [&]() -> td::Result> { + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + }, + [&](ton_api::validatorSession_compressedCandidateV2& c) { + res = [&]() -> td::Result> { + if (c.data_.size() > max_decompressed_data_size) { + return td::Status::Error("Compressed data is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + })); + return res; } td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, @@ -68,19 +92,28 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size) { - TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); - if (decompressed.size() != (size_t)decompressed_size) { - return td::Status::Error("decompressed size mismatch"); + bool improved_compression, + int decompressed_size, + int max_decompressed_size, + int proto_version) { + std::vector> roots; + if (!improved_compression) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed)); + } else { + TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size)); } - TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); if (roots.empty()) { return td::Status::Error("boc is empty"); } TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); - TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); - LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + int collated_data_mode = proto_version >= 5 ? 2 : 31; + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); + LOG(DEBUG) << "Decompressing block candidate " << (improved_compression ? "V2:" : ":") << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); } diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index e88376cd7..9ab53cc89 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -24,11 +24,15 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size); + int max_decompressed_data_size, + int proto_version); td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size); + bool improved_compression, + int decompressed_size, + int max_decompressed_size, + int proto_version); } // namespace ton::validatorsession diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 7147bf2d3..982d2642b 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -23,6 +23,8 @@ #include "adnl/adnl-node-id.hpp" #include "ton/ton-types.h" +#include + namespace ton { namespace validatorsession { @@ -71,26 +73,29 @@ struct ValidatorSessionNode { struct ValidatorSessionStats { enum { status_none = 0, status_received = 1, status_rejected = 2, status_approved = 3 }; + enum { recv_none = 0, recv_collated = 1, recv_broadcast = 2, recv_query = 3, recv_cached = 4, recv_startup = 5 }; struct Producer { - PublicKeyHash id = PublicKeyHash::zero(); - ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - double block_timestamp = -1.0; - td::Bits256 root_hash = td::Bits256::zero(); - td::Bits256 file_hash = td::Bits256::zero(); - std::string comment; - + PublicKeyHash validator_id = PublicKeyHash::zero(); + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; + td::Bits256 collated_data_hash = td::Bits256::zero(); bool is_accepted = false; bool is_ours = false; + double got_block_at = -1.0; + int got_block_by = 0; double got_submit_at = -1.0; - double collation_time = -1.0; - double validation_time = -1.0; - double collated_at = -1.0; - double validated_at = -1.0; + td::int32 gen_utime = -1; + std::string comment; + + double collation_time = -1.0, collated_at = -1.0; bool collation_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + double validation_time = -1.0, validated_at = -1.0; bool validation_cached = false; - double gen_utime = -1.0; std::vector approvers, signers; ValidatorWeight approved_weight = 0; @@ -129,20 +134,54 @@ struct ValidatorSessionStats { } } } + + tl_object_ptr tl() const { + std::string approvers_str(approvers.size(), '0'); + for (size_t i = 0; i < approvers.size(); ++i) { + approvers_str[i] = '0' + approvers[i]; + } + std::string signers_str(signers.size(), '0'); + for (size_t i = 0; i < signers.size(); ++i) { + signers_str[i] = '0' + signers[i]; + } + int flags = + (block_status != status_none || !candidate_id.is_zero() + ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK + : 0) | + (collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) | + (!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) | + (validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) | + (serialize_time >= 0.0 || deserialize_time >= 0.0 || serialized_size >= 0 + ? ton_api::validatorStats_stats_producer::Flags::SERIALIZE_TIME_MASK + : 0); + return create_tl_object( + flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id), + collated_data_hash, is_accepted, is_ours, got_block_at, got_block_by, got_submit_at, gen_utime, comment, + collation_time, collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, + validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str), + signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time, + serialized_size); + } }; struct Round { - double timestamp = -1.0; + double started_at = -1.0; std::vector producers; - }; - td::uint32 first_round; - std::vector rounds; + tl_object_ptr tl() const { + std::vector> producers_tl; + for (const auto &producer : producers) { + producers_tl.push_back(producer.tl()); + } + return create_tl_object(started_at, std::move(producers_tl)); + } + }; - bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; CatchainSeqno cc_seqno = 0; + bool success = false; double timestamp = -1.0; - PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; ValidatorWeight total_weight = 0; @@ -150,11 +189,36 @@ struct ValidatorSessionStats { ValidatorWeight signatures_weight = 0; td::uint32 approve_signatures = 0; ValidatorWeight approve_signatures_weight = 0; + + td::uint32 first_round = 0; + std::vector rounds; + + void fix_block_ids() { + for (auto &round : rounds) { + for (auto &producer : round.producers) { + producer.block_id.id = block_id.id; + } + } + } + + tl_object_ptr tl() const { + std::vector> rounds_tl; + for (const auto &round : rounds) { + rounds_tl.push_back(round.tl()); + } + int flags = success ? ton_api::validatorStats_stats::Flags::CREATOR_MASK : 0; + return create_tl_object( + flags, session_id, self.bits256_value(), create_tl_block_id(block_id), cc_seqno, success, timestamp, + creator.bits256_value(), total_validators, total_weight, signatures, signatures_weight, approve_signatures, + approve_signatures_weight, first_round, std::move(rounds_tl)); + } }; struct NewValidatorGroupStats { struct Node { PublicKeyHash id = PublicKeyHash::zero(); + PublicKey pubkey; + adnl::AdnlNodeIdShort adnl_id = adnl::AdnlNodeIdShort::zero(); ValidatorWeight weight = 0; }; @@ -162,9 +226,26 @@ struct NewValidatorGroupStats { ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; BlockSeqno last_key_block_seqno = 0; - double timestamp = -1.0; + double started_at = -1.0; + std::vector prev; td::uint32 self_idx = 0; - std::vector nodes; + PublicKeyHash self = PublicKeyHash::zero(); + std::vector nodes{}; + + tl_object_ptr tl() const { + std::vector> prev_arr; + for (const auto &p : prev) { + prev_arr.push_back(create_tl_block_id(p)); + } + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back(create_tl_object( + node.id.bits256_value(), node.pubkey.tl(), node.adnl_id.bits256_value(), node.weight)); + } + return create_tl_object( + session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, std::move(prev_arr), + self_idx, self.bits256_value(), std::move(nodes_arr)); + } }; struct EndValidatorGroupStats { @@ -175,13 +256,23 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; - std::vector nodes; + PublicKeyHash self = PublicKeyHash::zero(); + std::vector nodes{}; + + tl_object_ptr tl() const { + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + return create_tl_object(session_id, timestamp, self.bits256_value(), + std::move(nodes_arr)); + } }; struct BlockSourceInfo { - td::uint32 round, first_block_round; PublicKey source; - td::int32 source_priority; + BlockCandidatePriority priority; }; } // namespace validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3a913990c..27c105929 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -18,7 +18,6 @@ */ #include "validator-session.hpp" #include "td/utils/Random.h" -#include "td/utils/crypto.h" #include "candidate-serializer.h" #include "td/utils/overloaded.h" #include "ton/ton-tl.hpp" @@ -63,7 +62,7 @@ void ValidatorSessionImpl::process_blocks(std::vector auto to_approve = real_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &block : to_approve) { auto id = SentBlock::get_block_id(block); - if (approved_.count(id) && approved_[id].first <= td::Clocks::system()) { + if (approved_.contains(id) && approved_[id].first <= td::Clocks::system()) { msgs.emplace_back(create_tl_object( cur_round_, id, approved_[id].second.clone())); cnt++; @@ -222,14 +221,28 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast) { + bool is_overlay_broadcast, bool is_startup) { + bool is_optimistic = false; + ValidatorSessionCandidateId optimistic_prev_candidate = ValidatorSessionCandidateId::zero(); + if (is_overlay_broadcast) { + auto R = fetch_tl_object(data, true); + if (R.is_ok()) { + if (src == local_id()) { + return; + } + is_optimistic = true; + optimistic_prev_candidate = R.ok_ref()->prev_candidate_id_; + data = std::move(R.ok_ref()->data_); + } + } // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, - description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + description().opts().max_block_size + description().opts().max_collated_data_size + 1024, + description().opts().proto_version); double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) @@ -261,26 +274,85 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK || (block_round == 0 && is_optimistic)) { + VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id + << "]: bad round=" << block_round << " cur_round" << cur_round_; + return; + } + + BroadcastInfo broadcast_info{.candidate_id = block_id, + .received_at = td::Clocks::system(), + .deserialize_time = deserialize_time, + .serialized_size = data.size(), + .file_hash = file_hash, + .collated_data_hash = collated_data_file_hash}; + + if (is_optimistic) { + if (optimistic_broadcasts_.contains({block_round, src_idx})) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: duplicate optimistic broadcast for round " << block_round; + return; + } + int priority = description().get_node_priority(src_idx, block_round); + if (priority < 0) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: node is not a producer in round " << block_round; + return; + } + if (block_round > cur_round_) { + OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; + optimistic_broadcast.candidate = std::move(candidate); + optimistic_broadcast.prev_candidate_id = optimistic_prev_candidate; + optimistic_broadcast.broadcast_info = broadcast_info; + VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src + << ", round " << block_round; + validate_optimistic_broadcast( + BlockSourceInfo{description().get_source_public_key(src_idx), + BlockCandidatePriority{block_round, block_round, priority}}, + optimistic_broadcast.candidate->root_hash_, optimistic_broadcast.candidate->data_.clone(), + optimistic_broadcast.candidate->collated_data_.clone(), optimistic_broadcast.prev_candidate_id); + return; + } + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != + optimistic_prev_candidate) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: dropping optimistic broadcast for round " << block_round + << " - prev candidate mismatch"; + return; + } + } + process_received_block(block_round, src, src_idx, std::move(candidate), broadcast_info, is_overlay_broadcast, + is_startup); +} + +void ValidatorSessionImpl::process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, + const BroadcastInfo &info, bool is_overlay_broadcast, + bool is_startup) { + ValidatorSessionCandidateId block_id = info.candidate_id; auto stat = stats_get_candidate_stat(block_round, src, block_id); if (stat) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = info.received_at; + if (is_overlay_broadcast) { + stat->got_block_by = ValidatorSessionStats::recv_broadcast; + } else if (is_startup) { + stat->got_block_by = ValidatorSessionStats::recv_startup; + } else { + stat->got_block_by = ValidatorSessionStats::recv_query; + } } - stat->deserialize_time = deserialize_time; - stat->serialized_size = data.size(); - stat->root_hash = candidate->root_hash_; - stat->file_hash = file_hash; + stat->deserialize_time = info.deserialize_time; + stat->serialized_size = (int)info.serialized_size; + stat->block_id.root_hash = candidate->root_hash_; + stat->block_id.file_hash = info.file_hash; + stat->collated_data_hash = info.collated_data_hash; } - if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || - block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { - VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id - << "]: bad round=" << block_round << " cur_round" << cur_round_; - return; - } auto it = blocks_.find(block_id); if (it != blocks_.end()) { it->second->round_ = std::max(it->second->round_, block_round); @@ -300,16 +372,22 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } blocks_[block_id] = std::move(candidate); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { return; } - CHECK(!pending_approve_.count(block_id)); - CHECK(!approved_.count(block_id)); - CHECK(!pending_reject_.count(block_id)); - CHECK(!rejected_.count(block_id)); + CHECK(!pending_approve_.contains(block_id)); + CHECK(!approved_.contains(block_id)); + CHECK(!pending_reject_.contains(block_id)); + CHECK(!rejected_.contains(block_id)); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { @@ -320,6 +398,37 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } } +void ValidatorSessionImpl::validate_optimistic_broadcast(BlockSourceInfo source_info, + ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id) { + if (source_info.priority.round <= cur_round_) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : too old"; + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : wait for prev block"; + block_waiters_[prev_candidate_id].push_back( + [=, SelfId = actor_id(this), data = std::move(data), + collated_data = std::move(collated_data)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::validate_optimistic_broadcast, source_info, + root_hash, std::move(data), std::move(collated_data), prev_candidate_id); + } + }); + return; + } + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id(); + callback_->on_optimistic_candidate( + source_info, root_hash, std::move(data), std::move(collated_data), + description().get_source_public_key(description().get_source_idx(PublicKeyHash{it->second->src_})), + it->second->root_hash_, it->second->data_.clone(), it->second->collated_data_.clone()); +} + void ValidatorSessionImpl::process_message(PublicKeyHash src, td::BufferSlice data) { } @@ -412,7 +521,7 @@ void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSess stat->block_status = ValidatorSessionStats::status_approved; stat->comment = PSTRING() << "ts=" << ok_from; stat->validation_time = validation_time; - stat->gen_utime = (double)ok_from; + stat->gen_utime = (int)ok_from; stat->validated_at = td::Clocks::system(); stat->validation_cached = validation_cached; } @@ -451,44 +560,52 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } } -void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data, double collation_time, - bool collation_cached) { - if (data.size() > description().opts().max_block_size || - collated_data.size() > description().opts().max_collated_data_size) { - LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " - << collated_data.size(); +void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate c, double collation_time) { + if (c.candidate.data.size() > description().opts().max_block_size || + c.candidate.collated_data.size() > description().opts().max_collated_data_size) { + LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << c.candidate.data.size() << " " + << c.candidate.collated_data.size(); return; } - auto file_hash = sha256_bits256(data.as_slice()); - auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto file_hash = sha256_bits256(c.candidate.data.as_slice()); + auto collated_data_file_hash = sha256_bits256(c.candidate.collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), c.candidate.id.root_hash, file_hash, collated_data_file_hash); auto stat = stats_get_candidate_stat(round, local_id(), block_id); if (stat) { stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->block_timestamp = td::Clocks::system(); - stat->collation_cached = collation_cached; - stat->root_hash = root_hash; - stat->file_hash = file_hash; + if (auto it = sent_candidates_.find(block_id); it != sent_candidates_.end()) { + stat->got_block_at = it->second.sent_at; + } else { + stat->got_block_at = td::Clocks::system(); + } + stat->got_block_by = ValidatorSessionStats::recv_collated; + stat->collation_cached = c.is_cached; + stat->self_collated = c.self_collated; + stat->collator_node_id = c.collator_node_id; + stat->block_id = c.candidate.id; + stat->collated_data_hash = collated_data_file_hash; } if (round != cur_round_) { return; } - td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), - std::move(collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + SentCandidateStats &send_stats = send_candidate_broadcast(round, c.candidate); if (stat) { - stat->serialize_time = serialize_timer.elapsed(); - stat->serialized_size = B.size(); + stat->serialize_time = send_stats.serialize_time; + stat->serialized_size = send_stats.serialized_size; } - td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - - blocks_.emplace(block_id, std::move(b)); + blocks_[block_id] = create_tl_object( + local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), + std::move(c.candidate.collated_data)); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -496,6 +613,31 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan request_new_block(true); } +ValidatorSessionImpl::SentCandidateStats &ValidatorSessionImpl::send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate) { + ValidatorSessionCandidateId candidate_id = description().candidate_id( + local_idx(), candidate.id.root_hash, candidate.id.file_hash, candidate.collated_file_hash); + SentCandidateStats &stats = sent_candidates_[candidate_id]; + if (stats.sent) { + return stats; + } + stats.sent = true; + td::Timer serialize_timer; + auto b = create_tl_object( + local_id().tl(), round, candidate.id.root_hash, candidate.data.clone(), candidate.collated_data.clone()); + auto data = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + stats.serialize_time = serialize_timer.elapsed(); + stats.sent_at = td::Clocks::system(); + stats.serialized_size = data.size(); + if (optimistic_prev_candidate) { + data = create_serialize_tl_object( + 0, optimistic_prev_candidate.value(), std::move(data)); + } + td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(data)); + return stats; +} + void ValidatorSessionImpl::signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature) { if (round != cur_round_) { return; @@ -546,16 +688,15 @@ void ValidatorSessionImpl::check_generate_slot() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = std::move(R.ok_ref().candidate); - td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, R.move_as_ok(), + timer.elapsed()); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } }); - callback_->on_generate_slot( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(local_idx()), priority}, - std::move(P)); + callback_->on_generate_slot(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, priority}}, + std::move(P)); } else { alarm_timestamp().relax(t); } @@ -576,7 +717,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { return; } } - if (pending_approve_.count(block_id) || rejected_.count(block_id)) { + if (pending_approve_.contains(block_id) || rejected_.contains(block_id)) { return; } @@ -606,11 +747,13 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = td::Clocks::system(); + stat->got_block_by = ValidatorSessionStats::recv_cached; } - stat->root_hash = B->root_hash_; - stat->file_hash = td::sha256_bits256(B->data_); + stat->block_id.root_hash = B->root_hash_; + stat->block_id.file_hash = td::sha256_bits256(B->data_); + stat->collated_data_hash = td::sha256_bits256(B->collated_data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), @@ -634,11 +777,12 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { pending_approve_.insert(block_id); callback_->on_candidate( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}, + BlockSourceInfo{description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { - if (!active_requests_.count(block_id)) { + if (!active_requests_.contains(block_id)) { auto v = virtual_state_->get_block_approvers(description(), block_id); if (v.size() > 0) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); @@ -653,7 +797,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { << ": " << R.move_as_error(); } else { td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), - candidate_id, false); + candidate_id, false, false); } }); @@ -782,9 +926,9 @@ void ValidatorSessionImpl::check_all() { for (auto &B : to_approve) { if (B) { auto block_id = SentBlock::get_block_id(B); - auto pending = pending_approve_.count(block_id) == 1; - auto rejected = rejected_.count(block_id) == 1; - auto accepted = approved_.count(block_id) == 1; + auto pending = pending_approve_.contains(block_id); + auto rejected = rejected_.contains(block_id); + auto accepted = approved_.contains(block_id); sb << " " << block_id << " pending: " << pending << " rejected: " << rejected << " accepted: " << accepted << "\n"; } else { @@ -909,9 +1053,10 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { stats.rounds.pop_back(); } - BlockSourceInfo source_info{cur_round_, first_block_round_, - description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}; + BlockSourceInfo source_info{ + description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}; if (it == blocks_.end()) { callback_->on_block_committed(std::move(source_info), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), std::move(export_sigs), std::move(export_approve_sigs), @@ -931,7 +1076,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (round_idx >= cur_stats_.rounds.size()) { stats_add_round(); } - cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); + cur_stats_.rounds[round_idx].started_at = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -945,6 +1090,18 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { round_started_at_ = td::Timestamp::now(); round_debug_at_ = td::Timestamp::in(60.0); + for (auto it = optimistic_broadcasts_.begin(); it != optimistic_broadcasts_.end() && it->first.first <= cur_round_;) { + OptimisticBroadcast &optimistic_broadcast = it->second; + auto [block_round, src_idx] = it->first; + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) == + optimistic_broadcast.prev_candidate_id) { + process_received_block(block_round, description().get_source_id(src_idx), src_idx, + std::move(optimistic_broadcast.candidate), optimistic_broadcast.broadcast_info, true, + false); + } + it = optimistic_broadcasts_.erase(it); + } + check_all(); } @@ -966,8 +1123,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_candidate(broadcast, compress).move_as_ok(), td::optional(), - false); + serialize_candidate(broadcast, compress).move_as_ok(), + td::optional(), false, true); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), @@ -982,7 +1139,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) : unique_hash_(session_id) @@ -995,6 +1152,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; + allow_optimistic_generation_ = opts.proto_version >= 6; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } @@ -1032,9 +1190,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise pro promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); return; } - EndValidatorGroupStats stats; - stats.session_id = unique_hash_; - stats.timestamp = td::Clocks::system(); + EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system(), .self = local_id()}; stats.nodes.resize(description().get_total_nodes()); for (size_t i = 0; i < stats.nodes.size(); ++i) { stats.nodes[i].id = description().get_source_id(i); @@ -1107,7 +1263,6 @@ void ValidatorSessionImpl::start_up() { virtual_state_ = real_state_; check_all(); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, description().get_source_adnl_id(local_idx())); } void ValidatorSessionImpl::stats_init() { @@ -1143,7 +1298,7 @@ void ValidatorSessionImpl::stats_init() { if (cur_stats_.rounds.empty()) { stats_add_round(); } - cur_stats_.rounds[0].timestamp = td::Clocks::system(); + cur_stats_.rounds[0].started_at = td::Clocks::system(); stats_inited_ = true; } @@ -1156,13 +1311,13 @@ void ValidatorSessionImpl::stats_add_round() { td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { CHECK((size_t)priority < stat.producers.size()); - stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].validator_id = description().get_source_id(i); stat.producers[priority].is_ours = (local_idx() == i); stat.producers[priority].approvers.resize(description().get_total_nodes(), false); stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + while (!stat.producers.empty() && stat.producers.back().validator_id.is_zero()) { stat.producers.pop_back(); } } @@ -1177,7 +1332,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( } auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), - [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); + [&](const ValidatorSessionStats::Producer &p) { return p.validator_id == src; }); if (it == stats_round.producers.end()) { return nullptr; } @@ -1187,7 +1342,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( auto it2 = stats_pending_approve_.find({round, it->candidate_id}); if (it2 != stats_pending_approve_.end()) { for (td::uint32 node_id : it2->second) { - it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + process_approve(node_id, round, it->candidate_id); } stats_pending_approve_.erase(it2); } @@ -1227,19 +1382,16 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val obj.round_, description().get_source_id(node_id), candidate_id); if (stat && stat->got_submit_at <= 0.0) { stat->got_submit_at = td::Clocks::system(); + stat->block_id.root_hash = obj.root_hash_; + stat->block_id.file_hash = obj.file_hash_; + stat->collated_data_hash = obj.collated_data_file_hash_; } }, [&](const ton_api::validatorSession_message_approvedBlock &obj) { if (obj.candidate_ == skip_round_candidate_id()) { return; } - auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); - if (stat) { - stat->set_approved_by(node_id, description().get_node_weight(node_id), - description().get_total_weight()); - } else { - stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); - } + process_approve(node_id, obj.round_, obj.candidate_); }, [&](const ton_api::validatorSession_message_commit &obj) { if (obj.candidate_ == skip_round_candidate_id()) { @@ -1256,11 +1408,75 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } +void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, + ValidatorSessionCandidateId candidate_id) { + auto stat = stats_get_candidate_stat_by_id(round, candidate_id); + if (!stat) { + stats_pending_approve_[{round, candidate_id}].push_back(node_id); + return; + } + + bool was_approved_66pct = stat->approved_66pct_at > 0.0; + stat->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + bool is_approved_66pct = stat->approved_66pct_at > 0.0; + + if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && + cur_round_ == first_block_round_ && description().get_node_priority(local_idx(), round + 1) == 0 && + blocks_.contains(candidate_id) && + description().get_node_priority(description().get_source_idx(stat->validator_id), cur_round_) == 0) { + if (blocks_.contains(candidate_id)) { + generate_block_optimistic(round, candidate_id); + } else { + block_waiters_[candidate_id].push_back([=, SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generate_block_optimistic, round, candidate_id); + } + }); + } + } +} + +void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, + ValidatorSessionCandidateId prev_candidate_id) { + if (cur_round != cur_round_) { + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + return; + } + auto &block = it->second; + auto stat = stats_get_candidate_stat_by_id(cur_round, prev_candidate_id); + if (!stat) { + return; + } + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round + 1, cur_round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Optimistic generation error: " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, + &ValidatorSessionImpl::generated_optimistic_candidate, + cur_round + 1, R.move_as_ok(), prev_candidate_id); + }); +} + +void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate) { + if (cur_round_ > round) { + return; + } + send_candidate_broadcast(round, candidate.candidate, prev_candidate); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { return td::actor::create_actor("session", session_id, std::move(opts), local_id, std::move(nodes), std::move(callback), keyring, adnl, rldp, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index b099d65ec..f8e2d62e7 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -20,7 +20,7 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "ton/ton-types.h" @@ -78,11 +78,6 @@ class ValidatorSession : public td::actor::Actor { bool is_cached_ = false; }; - struct GeneratedCandidate { - BlockCandidate candidate; - bool is_cached = false; - }; - class Callback { public: virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data, @@ -98,6 +93,15 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) { + } + virtual void on_optimistic_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, PublicKey prev_source, + ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) { + } virtual ~Callback() = default; }; @@ -114,7 +118,7 @@ class ValidatorSession : public td::actor::Actor { catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); virtual ~ValidatorSession() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 690346f70..4861c52e1 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -76,6 +76,7 @@ class ValidatorSessionImpl : public ValidatorSession { std::map> blocks_; // src_round_candidate_[src_id][round] -> candidate id std::vector> src_round_candidate_; + std::map>> block_waiters_; catchain::CatChainSessionId unique_hash_; @@ -85,7 +86,7 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlay_manager_; td::actor::ActorOwn catchain_; std::unique_ptr description_; @@ -116,7 +117,7 @@ class ValidatorSessionImpl : public ValidatorSession { } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), - td::optional(), true); + td::optional(), true, false); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -161,6 +162,7 @@ class ValidatorSessionImpl : public ValidatorSession { bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; bool compress_block_candidates_ = false; + bool allow_optimistic_generation_ = false; ValidatorSessionStats cur_stats_; bool stats_inited_ = false; @@ -168,6 +170,30 @@ class ValidatorSessionImpl : public ValidatorSession { stats_pending_approve_; // round, candidate_id -> approvers std::map, std::vector> stats_pending_sign_; // round, candidate_id -> signers + + struct SentCandidateStats { + bool sent = false; + double sent_at = -1.0; + double serialize_time = -1.0; + size_t serialized_size = 0; + }; + std::map sent_candidates_; + + struct BroadcastInfo { + ValidatorSessionCandidateId candidate_id; + double received_at = -1.0; + double deserialize_time = -1.0; + size_t serialized_size = 0; + ValidatorSessionFileHash file_hash; + ValidatorSessionCollatedDataFileHash collated_data_hash; + }; + struct OptimisticBroadcast { + tl_object_ptr candidate; + ValidatorSessionCandidateId prev_candidate_id; + BroadcastInfo broadcast_info; + }; + std::map, OptimisticBroadcast> optimistic_broadcasts_; // round, src -> broadcast + void stats_init(); void stats_add_round(); ValidatorSessionStats::Producer *stats_get_candidate_stat( @@ -176,12 +202,16 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); + void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); + void generate_block_optimistic(td::uint32 cur_round, ValidatorSessionCandidateId prev_candidate_id); + void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); void start_up() override; void alarm() override; @@ -204,7 +234,13 @@ class ValidatorSessionImpl : public ValidatorSession { void preprocess_block(catchain::CatChainBlock *block); bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast); + bool is_overlay_broadcast, bool is_startup); + void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, const BroadcastInfo &info, + bool is_overlay_broadcast, bool is_startup); + void validate_optimistic_broadcast(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -217,8 +253,10 @@ class ValidatorSessionImpl : public ValidatorSession { void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); - void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated, double collation_time, bool collation_cached); + void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); + SentCandidateStats &send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate = {}); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 5f5544b22..e33010fe1 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -16,6 +14,8 @@ set(VALIDATOR_DB_SOURCE db/archive-slice.hpp db/celldb.cpp db/celldb.hpp + db/permanent-celldb/permanent-celldb-utils.cpp + db/permanent-celldb/permanent-celldb-utils.h db/files-async.hpp db/fileref.hpp db/fileref.cpp @@ -56,9 +56,17 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp - + storage-stat-cache.hpp + shard-block-verifier.hpp + shard-block-retainer.hpp + + collation-manager.hpp + collator-node/collator-node.hpp + collator-node/collator-node-session.hpp + collator-node/utils.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -74,8 +82,13 @@ set(VALIDATOR_HEADERS set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp + collation-manager.cpp + collator-node/collator-node.cpp + collator-node/collator-node-session.cpp + collator-node/utils.cpp get-next-key-blocks.cpp import-db-slice.cpp + import-db-slice-local.cpp shard-client.cpp state-serializer.cpp token-manager.cpp @@ -85,6 +98,9 @@ set(VALIDATOR_SOURCE validator-options.cpp queue-size-counter.cpp validator-telemetry.cpp + storage-stat-cache.cpp + shard-block-verifier.cpp + shard-block-retainer.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp @@ -153,6 +169,8 @@ set(FULL_NODE_SOURCE full-node-private-overlay.cpp full-node-serializer.hpp full-node-serializer.cpp + full-node-fast-sync-overlays.hpp + full-node-fast-sync-overlays.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index a35b74ce8..3c6a931cd 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -166,7 +166,7 @@ void ApplyBlock::written_block_data() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_, - std::move(P)); + true, std::move(P)); } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp new file mode 100644 index 000000000..7d694af98 --- /dev/null +++ b/validator/collation-manager.cpp @@ -0,0 +1,544 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collation-manager.hpp" + +#include "collator-node/collator-node.hpp" +#include "collator-node/utils.hpp" +#include "fabric.h" +#include "td/utils/Random.h" + +#include +#include +#include + +namespace ton::validator { + +void CollationManager::start_up() { + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); + update_collators_list(*opts_->get_collators_list()); + + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollationManager::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID), + std::make_unique(actor_id(this))); +} + +void CollationManager::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID)); +} + +void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + if (shard.is_masterchain()) { + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version); +} + +void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + if (shard.is_masterchain()) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + + auto& entry = optimistic_prev_cache_[prev_block_id]; + entry.block_data = std::move(prev_block); + ++entry.refcnt; + promise = [this, SelfId = actor_id(this), prev_block_id, + promise = std::move(promise)](td::Result R) mutable { + promise.set_result(std::move(R)); + td::actor::send_lambda_later(SelfId, [=, this]() { + auto it = optimistic_prev_cache_.find(prev_block_id); + CHECK(it != optimistic_prev_cache_.end()); + CHECK(it->second.refcnt > 0); + if (--it->second.refcnt == 0) { + optimistic_prev_cache_.erase(it); + } + }); + }; + + collate_shard_block(shard, min_masterchain_block_id, {prev_block_id}, creator, priority, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version, true); +} + +void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, td::Timestamp timeout, + int proto_version, bool is_optimistic) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + promise.set_error( + td::Status::Error(PSTRING() << "shard " << shard.to_str() << " is not configured in collators list")); + return; + } + + adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); + size_t selected_idx = 0; + for (int allow_banned = 0; allow_banned < 2; ++allow_banned) { + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (collator.banned_until && !allow_banned) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { + selected_collator = collator; + selected_idx = i; + break; + } + } + break; + } + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (check_collator(collator)) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } + } + break; + } + } + if (!selected_collator.is_zero() || s->self_collate) { + break; + } + } + + if (selected_collator.is_zero() && s->self_collate) { + td::Ref optimistic_prev_block; + if (is_optimistic) { + CHECK(prev.size() == 1); + TRY_RESULT_PROMISE_ASSIGN(promise, optimistic_prev_block, + create_block(prev[0], optimistic_prev_cache_.at(prev[0]).block_data.clone())); + } + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(optimistic_prev_block)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + + std::vector> prev_blocks; + BlockId next_block_id{shard, 0}; + for (const BlockIdExt& p : prev) { + prev_blocks.push_back(create_tl_block_id(p)); + next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); + } + + td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), + retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(GeneratedCandidate{.candidate = R.move_as_ok(), + .is_cached = false, + .self_collated = false, + .collator_node_id = selected_collator.bits256_value()}); + return; + } + if (!selected_collator.is_zero()) { + td::actor::send_closure(SelfId, &CollationManager::on_collate_query_error, selected_collator); + } + LOG(INFO) << "ERROR: collate query for " << next_block_id.to_str() << " to #" << selected_idx << " (" + << selected_collator << "): " << R.error(); + if (timeout < retry_at) { + promise.set_error(R.move_as_error()); + return; + } + delay_action( + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, + creator, priority, validator_set, max_answer_size, cancellation_token, + std::move(promise), timeout, proto_version, is_optimistic); + }, + retry_at); + }; + + if (selected_collator.is_zero()) { + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no suitable collator node")); + return; + } + + td::BufferSlice query; + if (is_optimistic) { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } else { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" + << selected_collator << ")"; + + td::Promise P2 = [=, SelfId = actor_id(this), P = std::move(P), + timer = td::Timer()](td::Result R) mutable { + TRY_RESULT_PROMISE_PREFIX(P, data, std::move(R), "rldp query failed: "); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + P.set_error(td::Status::Error(error->code_, error->message_)); + return; + } + TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(P, candidate, + deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); + if (candidate.pubkey.as_bits256() != creator.as_bits256()) { + P.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; + } + if (candidate.id.id != next_block_id) { + P.set_error(td::Status::Error("collate query: block id mismatch")); + return; + } + LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" + << selected_collator << ") in " << timer.elapsed() << "s"; + P.set_result(std::move(candidate)); + }; + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", + std::move(P2), timeout, std::move(query), max_answer_size); +} + +void CollationManager::update_options(td::Ref opts) { + auto old_list = opts_->get_collators_list(); + opts_ = std::move(opts); + auto list = opts_->get_collators_list(); + if (old_list != list) { + update_collators_list(*list); + } +} + +void CollationManager::validator_group_started(ShardIdFull shard) { + if (active_validator_groups_[shard]++ != 0) { + return; + } + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (s->active_cnt++ != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + alarm(); +} + +void CollationManager::validator_group_finished(ShardIdFull shard) { + if (--active_validator_groups_[shard] != 0) { + return; + } + active_validator_groups_.erase(shard); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (--s->active_cnt != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + --collator.active_cnt; + } + alarm(); +} + +void CollationManager::get_stats( + td::Promise> promise) { + auto stats = create_tl_object(); + stats->adnl_id_ = local_id_.bits256_value(); + for (ShardInfo& s : shards_) { + auto obj = create_tl_object(); + obj->shard_id_ = create_tl_shard_id(s.shard_id); + obj->active_ = s.active_cnt; + obj->self_collate_ = s.self_collate; + switch (s.select_mode) { + case CollatorsList::mode_random: + obj->select_mode_ = "random"; + break; + case CollatorsList::mode_ordered: + obj->select_mode_ = "ordered"; + break; + case CollatorsList::mode_round_robin: + obj->select_mode_ = "round_robin"; + break; + } + for (adnl::AdnlNodeIdShort& id : s.collators) { + obj->collators_.push_back(id.bits256_value()); + } + stats->shards_.push_back(std::move(obj)); + } + for (auto& [id, collator] : collators_) { + auto obj = create_tl_object(); + obj->adnl_id_ = id.bits256_value(); + obj->active_ = collator.active_cnt; + obj->alive_ = collator.alive; + if (collator.active_cnt && !collator.sent_ping) { + obj->ping_in_ = collator.ping_at.in(); + } else { + obj->ping_in_ = -1.0; + } + obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; + obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); + obj->banned_for_ = collator.banned_until ? collator.banned_until.in() : -1.0; + stats->collators_.push_back(std::move(obj)); + } + promise.set_value(std::move(stats)); +} + +void CollationManager::ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason) { + auto it = collators_.find(collator_id); + if (it == collators_.end()) { + return; + } + alarm_timestamp().relax(it->second.banned_until = td::Timestamp::in(BAN_DURATION)); + LOG(ERROR) << "Ban collator " << collator_id << " for " << BAN_DURATION << "s: " << reason; +} + +void CollationManager::update_collators_list(const CollatorsList& collators_list) { + shards_.clear(); + for (auto& [_, collator] : collators_) { + collator.active_cnt = 0; + } + auto old_collators = std::move(collators_); + collators_.clear(); + for (const auto& shard : collators_list.shards) { + shards_.push_back({.shard_id = shard.shard_id, + .select_mode = shard.select_mode, + .collators = shard.collators, + .self_collate = shard.self_collate}); + for (auto id : shard.collators) { + auto it = old_collators.find(id); + if (it == old_collators.end()) { + collators_[id]; + } else { + collators_[id] = std::move(it->second); + old_collators.erase(it); + } + } + } + for (auto& [shard, _] : active_validator_groups_) { + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + continue; + } + if (s->active_cnt++ != 0) { + continue; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + } + alarm(); +} + +CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull shard) { + for (auto& s : shards_) { + if (shard_intersects(shard, s.shard_id)) { + return &s; + } + } + return nullptr; +} + +void CollationManager::alarm() { + alarm_timestamp() = td::Timestamp::never(); + for (auto& [id, collator] : collators_) { + if (collator.banned_until) { + if (collator.banned_until.is_in_past()) { + collator.banned_until = {}; + LOG(ERROR) << "Unban collator " << id; + } else { + alarm_timestamp().relax(collator.banned_until); + } + } + if (collator.active_cnt == 0 || collator.sent_ping) { + continue; + } + if (collator.ping_at.is_in_past()) { + collator.sent_ping = true; + td::BufferSlice query = + create_serialize_tl_object(ton_api::collatorNode_pong::VERSION_MASK); + td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); + }; + LOG(DEBUG) << "sending ping to " << id; + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, id, "ping", std::move(P), + td::Timestamp::in(2.0), std::move(query)); + } else { + alarm_timestamp().relax(collator.ping_at); + } + } +} + +void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result R) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.sent_ping = false; + + auto r_pong = [&]() -> td::Result> { + TRY_RESULT(data, std::move(R)); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + return td::Status::Error(error->code_, error->message_); + } + return fetch_tl_object(data, true); + }(); + collator.last_ping_at = td::Timestamp::now(); + if (r_pong.is_error()) { + LOG(DEBUG) << "pong from " << id << " : " << r_pong.error(); + collator.alive = false; + collator.last_ping_status = r_pong.move_as_error(); + } else { + collator.alive = true; + collator.last_ping_status = td::Status::OK(); + auto pong = r_pong.move_as_ok(); + collator.version = pong->flags_ & ton_api::collatorNode_pong::VERSION_MASK ? pong->version_ : 0; + LOG(DEBUG) << "pong from " << id << " : OK, version=" << collator.version; + } + collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.ping_at = td::Timestamp::now(); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +void CollationManager::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + if (!collators_.contains(src)) { + promise.set_error(td::Status::Error("got request from unknown collator")); + return; + } + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)); + BlockIdExt block_id = create_block_id(query->block_id_); + auto it = optimistic_prev_cache_.find(block_id); + if (it == optimistic_prev_cache_.end()) { + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : not found"; + promise.set_error(td::Status::Error("block not found")); + return; + } + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : OK"; + promise.set_value( + serialize_tl_object(serialize_candidate(BlockCandidate(Ed25519_PublicKey{td::Bits256::zero()}, block_id, + td::Bits256::zero(), it->second.block_data.clone(), {}), + true), + true)); +} + +} // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp new file mode 100644 index 000000000..6691a387e --- /dev/null +++ b/validator/collation-manager.hpp @@ -0,0 +1,115 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp2/rldp.h" +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollationManager : public td::actor::Actor { + public: + CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), adnl_(adnl), rldp_(rldp) { + } + + void start_up() override; + void tear_down() override; + void alarm() override; + + void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, int proto_version); + + void collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); + + void update_options(td::Ref opts); + + void validator_group_started(ShardIdFull shard); + void validator_group_finished(ShardIdFull shard); + + void get_stats(td::Promise> promise); + + void ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason); + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + td::Timestamp timeout, int proto_version, bool is_optimistic = false); + + void update_collators_list(const CollatorsList& collators_list); + + struct CollatorInfo { + bool alive = false; + td::Timestamp ping_at = td::Timestamp::now(); + bool sent_ping = false; + size_t active_cnt = 0; + td::Timestamp last_ping_at = td::Timestamp::never(); + td::Status last_ping_status = td::Status::Error("not pinged"); + int version = -1; + // Collator is banned when in returns invalid block + td::Timestamp banned_until = td::Timestamp::never(); + }; + std::map collators_; + + struct ShardInfo { + ShardIdFull shard_id; + CollatorsList::SelectMode select_mode; + std::vector collators; + bool self_collate = false; + size_t cur_idx = 0; + + size_t active_cnt = 0; + }; + std::vector shards_; + + std::map active_validator_groups_; + + ShardInfo* select_shard_info(ShardIdFull shard); + void got_pong(adnl::AdnlNodeIdShort id, td::Result R); + void on_collate_query_error(adnl::AdnlNodeIdShort id); + + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + + struct OptimisticPrevCache { + td::BufferSlice block_data; + size_t refcnt = 0; + }; + std::map optimistic_prev_cache_; + + static constexpr double BAN_DURATION = 300.0; +}; + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp new file mode 100644 index 000000000..8e131f3c0 --- /dev/null +++ b/validator/collator-node/collator-node-session.cpp @@ -0,0 +1,314 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "collator-node-session.hpp" + +#include "collator-node.hpp" +#include "fabric.h" +#include "utils.hpp" + +namespace ton::validator { + +static BlockSeqno get_next_block_seqno(const std::vector& prev) { + if (prev.size() == 1) { + return prev[0].seqno() + 1; + } + CHECK(prev.size() == 2); + return std::max(prev[0].seqno(), prev[1].seqno()) + 1; +} + +CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, + td::Ref validator_set, BlockIdExt min_masterchain_block_id, + bool can_generate, Ref state, adnl::AdnlNodeIdShort local_id, + td::Ref opts, + td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : shard_(shard) + , prev_(std::move(prev)) + , validator_set_(validator_set) + , min_masterchain_block_id_(min_masterchain_block_id) + , can_generate_(can_generate) + , local_id_(local_id) + , opts_(opts) + , manager_(manager) + , adnl_(adnl) + , rldp_(rldp) + , next_block_seqno_(get_next_block_seqno(prev_)) { + update_masterchain_config(state); +} + +void CollatorNodeSession::start_up() { + LOG(INFO) << "Starting collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno() << ", next block seqno " << next_block_seqno_; + + if (can_generate_) { + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::tear_down() { + LOG(INFO) << "Finishing collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno(); + for (auto& [_, entry] : cache_) { + entry->cancel(td::Status::Error("validator session finished")); + } +} + +void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can_generate) { + CHECK(block_id.shard_full() == shard_); + can_generate_ = can_generate; + if (next_block_seqno_ > block_id.seqno()) { + return; + } + LOG(DEBUG) << "New shard block " << block_id.to_str(); + next_block_seqno_ = block_id.seqno() + 1; + prev_ = {block_id}; + + while (!cache_.empty()) { + auto& [cache_prev, entry] = *cache_.begin(); + if (entry->block_seqno < next_block_seqno_) { + entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << entry->block_seqno << " is too old, expected " + << next_block_seqno_)); + } else if (entry->block_seqno == next_block_seqno_ && prev_ != cache_prev) { + entry->cancel(td::Status::Error(PSTRING() << "invalid prev blocks for seqno " << entry->block_seqno)); + } else { + break; + } + if (!entry->has_external_query_at && entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (entry->has_external_query_at && !entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": somebody asked for block we didn't even try to generate"; + } + cache_.erase(cache_.begin()); + } + + if (can_generate_) { + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::update_masterchain_config(td::Ref state) { + ValidatorSessionConfig config = state->get_consensus_config(); + proto_version_ = config.proto_version; + max_candidate_size_ = config.max_block_size + config.max_collated_data_size + 1024; +} + +void CollatorNodeSession::generate_block(std::vector prev_blocks, + td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; + bool is_optimistic = o_optimistic_prev_block.not_null(); + BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); + if (next_block_seqno_ > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " + << next_block_seqno_)); + return; + } + if (next_block_seqno_ == block_seqno && prev_ != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } + if (next_block_seqno_ + 10 < block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too new, current is " + << next_block_seqno_)); + return; + } + + static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, + BlockSeqno block_seqno, const td::optional& o_priority, + bool is_optimistic) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority.value().round - o_priority.value().first_block_round + << ",priority=" << o_priority.value().priority; + sb << ",first_block_round=" << o_priority.value().first_block_round; + sb << "}"; + } else { + sb << " internal"; + } + if (is_optimistic) { + sb << " opt"; + } + }; + auto prefix = [&](td::StringBuilder& sb) { + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority, is_optimistic); + }; + + auto cache_entry = cache_[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = cache_[prev_blocks] = std::make_shared(); + } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at - cache_entry->has_internal_query_at + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at - cache_entry->has_external_query_at + << "s after external query [LOST]"; + }; + } + } + if (cache_entry->result) { + auto has_result_ago = td::Timestamp::now() - cache_entry->has_result_at; + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + + if (cache_entry->started) { + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; + return; + } + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev_blocks), + .validator_set = validator_set_, + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true, + .optimistic_prev_block = o_optimistic_prev_block}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), + timer = td::Timer{}](td::Result R) mutable { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority, is_optimistic); + sb << ": " << (R.is_ok() ? "OK" : R.error().to_string()) << " time=" << timer.elapsed(); + }; + td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); + }); +} + +void CollatorNodeSession::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); +} + +void CollatorNodeSession::process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, + BlockCandidatePriority priority, bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (is_optimistic) { + if (prev_blocks.size() != 1) { + promise.set_error(td::Status::Error("optimistic collation, expected 1 prev block")); + return; + } + auto it = cache_.find(prev_blocks); + if (it == cache_.end() || it->second->started) { + BlockIdExt prev_block = prev_blocks[0]; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, prev_block, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont, src, prev_block, + priority, timeout, std::move(promise), std::move(R)); + }); + return; + } + } + generate_block(std::move(prev_blocks), priority, {}, timeout, std::move(promise)); +} + +void CollatorNodeSession::process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data) { + if (prev_block_data.is_ok()) { + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, prev_block_data.move_as_ok()), + "invalid prev block data in db: "); + LOG(INFO) << "got prev block from db for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); + return; + } + td::actor::send_closure( + rldp_, &rldp2::Rldp::send_query_ex, local_id_, src, "getprevblock", + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont2, prev_block_id, priority, + timeout, std::move(promise), std::move(R)); + }, + timeout, + create_serialize_tl_object(0, create_tl_block_id(prev_block_id)), + max_candidate_size_); +} + +void CollatorNodeSession::process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R) { + TRY_RESULT_PROMISE_PREFIX(promise, response, std::move(R), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, f, fetch_tl_object(response, true), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, candidate, + deserialize_candidate(std::move(f), max_candidate_size_, proto_version_), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, std::move(candidate.data)), + "invalid prev block data from validator: "); + LOG(INFO) << "got prev block from validator for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); +} + +void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp new file mode 100644 index 000000000..0f9f03d30 --- /dev/null +++ b/validator/collator-node/collator-node-session.hpp @@ -0,0 +1,95 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollatorNodeSession : public td::actor::Actor { + public: + CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, + BlockIdExt min_masterchain_block_id, bool can_generate, td::Ref state, + adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp); + + void start_up() override; + void tear_down() override; + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); + + void process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); + void update_masterchain_config(td::Ref state); + + private: + ShardIdFull shard_; + std::vector prev_; + td::Ref validator_set_; + BlockIdExt min_masterchain_block_id_; + bool can_generate_; + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + struct CacheEntry { + bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; + BlockSeqno block_seqno = 0; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + void cancel(td::Status reason); + }; + + BlockSeqno next_block_seqno_; + std::map, std::shared_ptr> cache_; + + td::uint32 proto_version_ = 0; + td::uint32 max_candidate_size_ = 0; + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise); + void process_result(std::shared_ptr cache_entry, td::Result R); + + void process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data); + void process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R); +}; + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp new file mode 100644 index 000000000..9b03d9dc0 --- /dev/null +++ b/validator/collator-node/collator-node.cpp @@ -0,0 +1,511 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collator-node.hpp" +#include "ton/ton-tl.hpp" +#include "fabric.h" +#include "block-auto.h" +#include "block-db.h" +#include "td/utils/lz4.h" +#include "checksum.h" +#include "impl/collator-impl.h" +#include "impl/shard.hpp" +#include "utils.hpp" + +namespace ton::validator { + +CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { +} + +void CollatorNode::start_up() { + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollatorNode::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); +} + +void CollatorNode::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); +} + +void CollatorNode::add_shard(ShardIdFull shard) { + CHECK(shard.is_valid_ext() && !shard.is_masterchain()); + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { + return; + } + LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); + collating_shards_.push_back(shard); + if (last_masterchain_state_.is_null()) { + return; + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (validator_group.actor.empty() && shard_intersects(shard, group_shard)) { + validator_group.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, + last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), + can_generate(), last_masterchain_state_, local_id_, opts_, manager_, adnl_, rldp_); + } + } +} + +void CollatorNode::del_shard(ShardIdFull shard) { + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + if (it != collating_shards_.end()) { + collating_shards_.erase(it); + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (!validator_group.actor.empty() && shard_intersects(shard, group_shard) && !can_collate_shard(group_shard)) { + validator_group.actor = {}; + } + } +} + +void CollatorNode::update_options(td::Ref opts) { + for (auto& [_, shard] : validator_groups_) { + if (!shard.actor.empty()) { + td::actor::send_closure(shard.actor, &CollatorNodeSession::update_options, opts); + } + } + opts_ = std::move(opts); +} + +void CollatorNode::new_masterchain_block_notification(td::Ref state) { + last_masterchain_state_ = state; + + if (state->last_key_block_id().seqno() != last_key_block_seqno_) { + last_key_block_seqno_ = state->last_key_block_id().seqno(); + mc_config_status_ = check_mc_config(); + if (mc_config_status_.is_error()) { + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; + } + + validator_adnl_ids_.clear(); + for (int next : {-1, 0, 1}) { + td::Ref vals = state->get_total_validator_set(next); + if (vals.not_null()) { + for (const ValidatorDescr& descr : vals->export_vector()) { + if (descr.addr.is_zero()) { + validator_adnl_ids_.insert( + adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); + } else { + validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr)); + } + } + } + } + for (auto& [_, group] : validator_groups_) { + if (!group.actor.empty()) { + td::actor::send_closure(group.actor, &CollatorNodeSession::update_masterchain_config, state); + } + } + } + + std::map> new_shards; + for (auto& v : state->get_shards()) { + auto shard = v->shard(); + if (v->before_split()) { + CHECK(!v->before_merge()); + new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()}); + new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()}); + } else if (v->before_merge()) { + ShardIdFull p_shard = shard_parent(shard); + auto it = new_shards.find(p_shard); + if (it == new_shards.end()) { + new_shards.emplace(p_shard, std::vector(2)); + } + bool left = shard_child(p_shard.shard, true) == shard.shard; + new_shards[p_shard][left ? 0 : 1] = v->top_block_id(); + } else { + new_shards.emplace(shard, std::vector{v->top_block_id()}); + } + } + for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { + if (new_shards.contains(it->first)) { + ++it; + } else { + it = validator_groups_.erase(it); + } + } + for (auto& [shard, prev] : new_shards) { + auto validator_set = state->get_validator_set(shard); + CatchainSeqno cc_seqno = validator_set->get_catchain_seqno(); + auto [it, created] = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + it->second.prev = std::move(prev); + if (created || it->second.cc_seqno != cc_seqno) { + it->second.cc_seqno = cc_seqno; + if (can_collate_shard(shard)) { + it->second.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, + last_masterchain_state_->get_block_id(), can_generate(), last_masterchain_state_, local_id_, opts_, + manager_, adnl_, rldp_); + } + } else if (!it->second.actor.empty() && prev.size() == 1) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], + can_generate()); + } + auto it2 = future_validator_groups_.find({shard, cc_seqno}); + if (it2 != future_validator_groups_.end()) { + FutureValidatorGroup& future_group = it2->second; + if (!it->second.actor.empty()) { + for (const BlockIdExt& block_id : future_group.pending_blocks) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, + can_generate()); + } + } + for (auto& promise : future_group.promises) { + promise.set_value(td::Unit()); + } + future_validator_groups_.erase(it2); + } + } + + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) { + if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { + ++it; + } else { + auto& future_group = it->second; + for (auto& promise : future_group.promises) { + promise.set_error(td::Status::Error("validator group is outdated")); + } + it = future_validator_groups_.erase(it); + } + } +} + +void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { + shard_client_handle_ = shard_client_handle; +} + +void CollatorNode::new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno) { + if (!can_collate_shard(block_id.shard_full())) { + return; + } + auto it = validator_groups_.find(block_id.shard_full()); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + auto future_group = get_future_validator_group(block_id.shard_full(), cc_seqno); + if (future_group.is_error()) { + LOG(DEBUG) << "Dropping new shard block " << block_id.to_str() << " cc_seqno=" << cc_seqno << " : " + << future_group.error(); + } else { + LOG(DEBUG) << "New shard block in future validator group " << block_id.to_str() << " cc_seqno=" << cc_seqno; + future_group.ok()->pending_blocks.push_back(block_id); + } + return; + } + if (!it->second.actor.empty()) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, can_generate()); + } +} + +td::Result CollatorNode::get_future_validator_group(ShardIdFull shard, + CatchainSeqno cc_seqno) { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() && shard.pfx_len() != 0) { + it = validator_groups_.find(shard_parent(shard)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, true)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, false)); + } + if (it == validator_groups_.end()) { + return td::Status::Error("no such shard"); + } + if (cc_seqno < it->second.cc_seqno) { // past validator group + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is outdated (current is " << it->second.cc_seqno << ")"); + } + if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is too big (currently known is " << it->second.cc_seqno << ")"); + } + // future validator group + return &future_validator_groups_[{shard, cc_seqno}]; +} + +static td::BufferSlice serialize_error(td::Status error) { + return create_serialize_tl_object(error.code(), error.message().c_str()); +} + +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, + td::uint32& val_set_hash) { + CHECK(!block.id.is_masterchain()); + if (block.pubkey == creator) { + return block; + } + auto root = vm::std_boc_deserialize(block.data).move_as_ok(); + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk)); + CHECK(tlb::unpack_cell(blk.extra, extra)); + CHECK(tlb::unpack_cell(blk.info, info)); + extra.created_by = creator.as_bits256(); + CHECK(tlb::pack_cell(blk.extra, extra)); + CHECK(tlb::pack_cell(root, blk)); + block.data = vm::std_boc_serialize(root, 31).move_as_ok(); + + block.id.root_hash = root->get_hash().bits(); + block.id.file_hash = block::compute_file_hash(block.data.as_slice()); + block.pubkey = creator; + + cc_seqno = info.gen_catchain_seqno; + val_set_hash = info.gen_validator_list_hash_short; + + for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { + auto block_state_proof = create_block_state_proof(root).move_as_ok(); + + auto& broadcast = broadcast_ref.write(); + broadcast.block_id = block.id; + broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); + } + return block; +} + +void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + promise = [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + if (R.error().code() == ErrorCode::timeout) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(serialize_error(R.move_as_error())); + } + } else { + promise.set_result(R.move_as_ok()); + } + }; + if (!opts_->check_collator_node_whitelist(src)) { + promise.set_error(td::Status::Error("not authorized")); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error("src is not a validator")); + return; + } + auto r_ping = fetch_tl_object(data, true); + if (r_ping.is_ok()) { + process_ping(src, *r_ping.ok_ref(), std::move(promise)); + return; + } + + bool is_optimistic = false; + ShardIdFull shard; + CatchainSeqno cc_seqno; + std::vector prev_blocks; + BlockCandidatePriority priority; + Ed25519_PublicKey creator; + if (auto R = fetch_tl_object(data, true); R.is_ok()) { + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else if (auto R = fetch_tl_object(data, true); R.is_ok()) { + is_optimistic = true; + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else { + promise.set_error(td::Status::Error("cannot parse request")); + return; + } + td::Promise new_promise = [promise = std::move(promise), src, + shard](td::Result R) mutable { + if (R.is_error()) { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": error: " << R.error(); + promise.set_error(R.move_as_error()); + } else { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": success"; + promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); + } + }; + new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_, + manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + + CollatorNodeResponseStats stats; + stats.self = local_id.pubkey_hash(); + stats.validator_id = PublicKey(pubkeys::Ed25519(creator)).compute_short_id(); + stats.original_block_id = block.id; + stats.collated_data_hash = block.collated_file_hash; + + CatchainSeqno cc_seqno; + td::uint32 val_set_hash; + block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); + + stats.block_id = block.id; + stats.timestamp = td::Clocks::system(); + td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats)); + + td::Promise P = + new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, + val_set_hash, std::move(P)); + }; + if (!shard.is_valid_ext()) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (prev_blocks.size() != 1 && prev_blocks.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); + return; + } + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << (is_optimistic ? ", optimistic" : ""); + process_generate_block_query(src, shard, cc_seqno, std::move(prev_blocks), priority, is_optimistic, + td::Timestamp::in(10.0), std::move(new_promise)); +} + +void CollatorNode::process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); + return; + } + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); + return; + } + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); + future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, src, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), is_optimistic, timeout, std::move(promise)); + }); + return; + } + ValidatorGroupInfo& validator_group_info = it->second; + if (validator_group_info.actor.empty()) { + promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); + return; + } + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::process_request, src, + std::move(prev_blocks), priority, is_optimistic, timeout, std::move(promise)); +} + +td::Status CollatorNode::check_out_of_sync() { + if (last_masterchain_state_.is_null() || !shard_client_handle_) { + return td::Status::Error("not inited"); + } + auto now = (UnixTime)td::Clocks::system(); + if (last_masterchain_state_->get_unix_time() < now - 60 || shard_client_handle_->unix_time() < now - 60) { + return td::Status::Error(PSTRING() << "out of sync: mc " << now - last_masterchain_state_->get_unix_time() + << "s ago, shardclient " << now - shard_client_handle_->unix_time() << "s ago"); + } + return td::Status::OK(); +} + +td::Status CollatorNode::check_mc_config() { + if (last_masterchain_state_.is_null()) { + return td::Status::Error("not inited"); + } + TRY_RESULT_PREFIX( + config, + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), last_masterchain_state_->get_block_id(), + block::ConfigInfo::needCapabilities), + "cannot unpack masterchain config"); + if (config->get_global_version() > Collator::supported_version()) { + return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() + << " (supported: " << Collator::supported_version() << ")"); + } + if (config->get_capabilities() & ~Collator::supported_capabilities()) { + return td::Status::Error(PSTRING() << "unsupported capabilities " << config->get_capabilities() + << " (supported: " << Collator::supported_capabilities() << ")"); + } + td::Status S = td::Status::OK(); + config->foreach_config_param([&](int idx, td::Ref param) { + if (idx < 0) { + return true; + } + if (!block::gen::ConfigParam{idx}.validate_ref(1024, std::move(param))) { + S = td::Status::Error(PSTRING() << "unknown ConfigParam " << idx); + return false; + } + return true; + }); + return S; +} + +void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, + td::Promise promise) { + LOG(DEBUG) << "got ping from " << src; + TRY_STATUS_PROMISE(promise, check_out_of_sync()); + TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); + auto pong = create_tl_object(); + if (ping.flags_ & ton_api::collatorNode_pong::VERSION_MASK) { + pong->flags_ |= ton_api::collatorNode_pong::VERSION_MASK; + pong->version_ = COLLATOR_NODE_VERSION; + } + promise.set_result(serialize_tl_object(pong, true)); +} + +bool CollatorNode::can_collate_shard(ShardIdFull shard) const { + return std::any_of(collating_shards_.begin(), collating_shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); +} + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node.hpp b/validator/collator-node/collator-node.hpp new file mode 100644 index 000000000..cb9cec47c --- /dev/null +++ b/validator/collator-node/collator-node.hpp @@ -0,0 +1,96 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "collator-node-session.hpp" +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollatorNode : public td::actor::Actor { + public: + CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp); + void start_up() override; + void tear_down() override; + void add_shard(ShardIdFull shard); + void del_shard(ShardIdFull shard); + + void update_options(td::Ref opts); + + void new_masterchain_block_notification(td::Ref state); + void update_shard_client_handle(BlockHandle shard_client_handle); + void new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno); + + private: + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); + void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); + + bool can_collate_shard(ShardIdFull shard) const; + + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + std::vector collating_shards_; + std::set validator_adnl_ids_; + + struct ValidatorGroupInfo { + CatchainSeqno cc_seqno{0}; + std::vector prev; + td::actor::ActorOwn actor; + }; + struct FutureValidatorGroup { + std::vector pending_blocks; + std::vector> promises; + }; + std::map validator_groups_; + std::map, FutureValidatorGroup> future_validator_groups_; + + td::Ref last_masterchain_state_; + BlockHandle shard_client_handle_; + + td::Status mc_config_status_ = td::Status::Error("not inited"); + BlockSeqno last_key_block_seqno_ = (BlockSeqno)-1; + + td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); + + td::Status check_out_of_sync(); + td::Status check_mc_config(); + + bool can_generate() { + return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); + } + + static constexpr int COLLATOR_NODE_VERSION = 1; + + public: + static constexpr int VERSION_OPTIMISTIC_COLLATE = 1; +}; + +} // namespace ton::validator diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp new file mode 100644 index 000000000..23e2ec40f --- /dev/null +++ b/validator/collator-node/utils.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "utils.hpp" +#include "checksum.h" +#include "keys/keys.hpp" +#include "ton/ton-tl.hpp" +#include "validator-session/candidate-serializer.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); + } + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); +} + +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version) { + td::Result res; + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); + return res; +} + +} // namespace ton::validator diff --git a/crypto/ellcurve/Fp25519.h b/validator/collator-node/utils.hpp similarity index 63% rename from crypto/ellcurve/Fp25519.h rename to validator/collator-node/utils.hpp index b6b63be5e..26b67d5a9 100644 --- a/crypto/ellcurve/Fp25519.h +++ b/validator/collator-node/utils.hpp @@ -13,20 +13,15 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "common/refcnt.hpp" -#include "openssl/residue.h" - -namespace ellcurve { -using namespace arith; -// returns 2^255-19 -const Bignum& P25519(); +#include "ton/ton-types.h" +#include "tl/generate/auto/tl/ton_api.h" -// residue ring modulo P25519 -td::Ref Fp25519(); +namespace ton::validator { -} // namespace ellcurve +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version); +} // namespace ton::validator diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 8c7cde170..d8afa1c0b 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -311,10 +311,15 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: } void ArchiveManager::register_perm_state(FileReferenceShort id) { - BlockSeqno masterchain_seqno = 0; - id.ref().visit(td::overloaded( - [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); - perm_states_[{masterchain_seqno, id.hash()}] = id; + td::uint64 size; + auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << id.filename_short() << " : " << r_stat.move_as_error(); + size = 0; + } else { + size = r_stat.ok().size_; + } + perm_states_[{id.seqno_of_persistent_state(), id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -339,17 +344,37 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t .release(); } -void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, +namespace { + +FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc_block_id, PersistentStateType type) { + FileReferenceShort result; + type.visit(td::overloaded( + [&](UnsplitStateType const &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); }, + [&](SplitAccountStateType const &account_state) { + result = fileref::SplitAccountState::create(block_id, mc_block_id, account_state.effective_shard_id); + }, + [&](SplitPersistentStateType const &persistent_state) { + result = fileref::SplitPersistentState::create(block_id, mc_block_id); + })); + return result; +} + +} // namespace + +void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { @@ -357,23 +382,21 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma std::move(write_state), std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_impl( - BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + FileReferenceShort const &id, td::Promise promise, std::function)> create_writer) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; - BlockSeqno masterchain_seqno = masterchain_block_id.seqno(); - auto hash = id.hash(); - if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) { + if (perm_states_.find({id.seqno_of_persistent_state(), id.hash()}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } auto path = db_root_ + "/archive/states/" + id.filename_short(); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result R) mutable { + [SelfId = actor_id(this), id = id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -417,7 +440,7 @@ void ArchiveManager::get_previous_persistent_state_files( BlockSeqno mc_seqno = it->first.first; std::vector> files; while (it->first.first == mc_seqno) { - files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + files.emplace_back(db_root_ + "/archive/states/" + it->second.id.filename_short(), it->second.id.shard()); if (it == perm_states_.begin()) { break; } @@ -427,8 +450,8 @@ void ArchiveManager::get_previous_persistent_state_files( } void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -439,9 +462,10 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master td::actor::create_actor("readfile", path, 0, -1, 0, std::move(promise)).release(); } -void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; +void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -452,15 +476,16 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); } -void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; +void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); - if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { - promise.set_result(false); + auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); + if (it == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready)); return; } - promise.set_result(true); + promise.set_result(it->second.size); } void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, @@ -909,6 +934,11 @@ void ArchiveManager::start_up() { R = FileReferenceShort::create(newfname); R.ensure(); } + if (!R.ok().is_state_like()) { + LOG(ERROR) << "deleting file that is not state-like '" << fname << "'"; + td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore(); + return; + } register_perm_state(R.move_as_ok()); } }).ensure(); @@ -956,8 +986,8 @@ void ArchiveManager::alarm() { } } -void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { - auto p = get_temp_package_id_by_unixtime(mc_ts - TEMP_PACKAGES_TTL); +void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { + auto p = get_temp_package_id_by_unixtime((double)mc_ts - TEMP_PACKAGES_TTL); std::vector vec; for (auto &x : temp_files_) { if (x.first < p) { @@ -985,7 +1015,7 @@ void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl if (it == desc.first_blocks.end()) { continue; } - if (it->second.ts < gc_ts - archive_ttl) { + if ((double)it->second.ts < (double)gc_ts - archive_ttl) { vec.push_back(f.first); } } @@ -1023,15 +1053,23 @@ void ArchiveManager::persistent_state_gc(std::pair last) { int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, - [&](const fileref::PersistentStateShort &x) { - res = 0; - seqno = x.masterchain_seqno; - }, - [&](const auto &obj) { res = -1; })); + F.id.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const fileref::SplitAccountState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const fileref::SplitPersistentState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); if (res == -1) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } if (res != 0) { @@ -1081,7 +1119,7 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } delay_action( @@ -1202,12 +1240,7 @@ void ArchiveManager::prepare_stats(td::Promise states; for (auto &[key, file] : perm_states_) { BlockSeqno seqno = key.first; - auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); - if (r_stat.is_error()) { - LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); - } else { - states[seqno] += r_stat.move_as_ok().size_; - } + states[seqno] += file.size; } td::StringBuilder sb; for (auto &[seqno, size] : states) { @@ -1220,6 +1253,15 @@ void ArchiveManager::prepare_stats(td::Promise f) { + for (auto &[_, file] : temp_files_) { + if (file.deleted) { + continue; + } + td::actor::send_closure(file.file_actor_id(), &ArchiveSlice::iterate_block_handles, f); + } +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; @@ -1308,16 +1350,18 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han auto it = perm_states_.begin(); while (it != perm_states_.end()) { int res = 0; - it->second.ref().visit(td::overloaded( + it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitPersistentState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitAccountState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); if (res <= 0) { it++; } else { auto it2 = it; it++; - td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + it2->second.id.filename_short()).ignore(); perm_states_.erase(it2); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index d919e32ee..0d0af4095 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -19,6 +19,7 @@ #pragma once #include "archive-slice.hpp" +#include "interfaces/persistent-state.h" namespace ton { @@ -43,16 +44,17 @@ class ArchiveManager : public td::actor::Actor { void get_file(ConstBlockHandle handle, FileReference ref_id, td::Promise promise); void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); - void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, - td::Promise promise); - void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, - td::Promise promise); + void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice data, td::Promise promise); + void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_state, td::Promise promise); void get_zero_state(BlockIdExt block_id, td::Promise promise); - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise); - void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise); + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_size, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, td::Promise>> promise); @@ -60,7 +62,7 @@ class ArchiveManager : public td::actor::Actor { void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl); + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl); /* from LTDB */ void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); @@ -83,6 +85,8 @@ class ArchiveManager : public td::actor::Actor { void prepare_stats(td::Promise>> promise); + void iterate_temp_block_handles(std::function f); + static constexpr td::uint32 archive_size() { return 20000; } @@ -189,7 +193,11 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state + struct PermState { + FileReferenceShort id; + td::uint64 size; + }; + std::map, PermState> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); @@ -214,7 +222,7 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); - void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + void add_persistent_state_impl(FileReferenceShort const &id, td::Promise promise, std::function)> create_writer); void register_perm_state(FileReferenceShort id); @@ -235,7 +243,7 @@ class ArchiveManager : public td::actor::Actor { void update_permanent_slices(); - static const td::uint32 TEMP_PACKAGES_TTL = 3600; + static constexpr double TEMP_PACKAGES_TTL = 3600; }; } // namespace validator diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp deleted file mode 100644 index aa89570d9..000000000 --- a/validator/db/archive-mover.cpp +++ /dev/null @@ -1,493 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "archive-mover.hpp" -#include "td/actor/MultiPromise.h" -#include "validator/fabric.h" - -namespace ton { - -namespace validator { - -void ArchiveFileMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle0(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - CHECK(handle_->moved_to_archive()); - CHECK(handle_->handle_moved_to_archive()); - finish_query(); - return; - } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle1(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P)); -} - -void ArchiveFileMover::got_block_handle2(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - finish_query(); -} - -void ArchiveFileMover::got_block_handle() { - if (!handle_->is_applied()) { - finish_query(); - return; - } - if (handle_->id().seqno() == 0) { - processed_all_children(); - return; - } - - CHECK(handle_->inited_prev()); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_child() { - if (!left_ || !handle_->merge_before()) { - processed_all_children(); - return; - } - left_ = false; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_all_children() { - if (!handle_->received()) { - got_block_data(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_data(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - data_ = R.move_as_ok(); - } - if (!handle_->inited_proof()) { - got_block_proof(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_ = R.move_as_ok(); - } - if (!handle_->inited_proof_link()) { - got_block_proof_link(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof_link(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_link_ = R.move_as_ok(); - } - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_data); - }); - - if (data_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_}, - std::move(data_), ig.get_promise()); - } - if (proof_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_}, - std::move(proof_), ig.get_promise()); - } - if (proof_link_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_}, - std::move(proof_link_), ig.get_promise()); - } -} - -void ArchiveFileMover::written_data() { - handle_->set_moved_to_archive(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P)); -} - -void ArchiveFileMover::written_handle() { - CHECK(handle_->handle_moved_to_archive()); - finish_query(); -} - -void ArchiveFileMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveFileMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveKeyBlockMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0); - } - }); - if (proof_link_) { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof0() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof1() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof2() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3); - } - }); - if (proof_link_) { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof3() { - if (proof_link_) { - written_data(); - } else { - proof_link_ = true; - start_up(); - } -} - -void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) { - data_ = std::move(data); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data); - }); - - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, - fileref::ProofLink{block_id_}, std::move(data_), std::move(P)); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_}, - std::move(data_), std::move(P)); - } -} - -void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) { - data_ = std::move(data); - written_data(); -} - -void ArchiveKeyBlockMover::written_data() { - td::Ref proof_link; - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } - auto ts = proof_link->get_basic_header_info().move_as_ok().utime; - auto te = ValidatorManager::persistent_state_ttl(ts); - if (te < td::Clocks::system()) { - finish_query(); - return; - } -} - -void ArchiveKeyBlockMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveKeyBlockMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks); - } - }); - td::actor::create_actor("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(), - old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(), - std::move(P)) - .release(); -} - -void ArchiveMover::moved_blocks() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok()); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P)); -} - -void ArchiveMover::got_handle(BlockHandle handle) { - handle_ = std::move(handle); - CHECK(handle_->is_applied()); - CHECK(handle_->inited_state_boc()); - CHECK(!handle_->deleted_state_boc()); - auto P = td::PromiseCreator::lambda( - [handle = handle_, SelfId = actor_id(this)](td::Result> R) mutable { - R.ensure(); - auto S = create_shard_state(handle->id(), R.move_as_ok()); - S.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); - }); - td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P)); -} - -void ArchiveMover::got_state(td::Ref state) { - state_ = std::move(state); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::moved_key_blocks() { - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::run() { - if (to_move_.empty() && to_check_.empty()) { - completed(); - return; - } - if (!to_check_.empty()) { - auto B = to_check_.back(); - CHECK(to_check_set_.count(B) == 1); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P)); - return; - } - CHECK(!to_move_.empty()); -} - -void ArchiveMover::got_to_check_handle(td::Result R) { - if (R.is_error()) { - CHECK(R.error().code() == ErrorCode::notready); - run(); - return; - } - auto handle = R.move_as_ok(); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/archive-mover.hpp b/validator/db/archive-mover.hpp deleted file mode 100644 index be0d8dc75..000000000 --- a/validator/db/archive-mover.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "filedb.hpp" -#include "blockdb.hpp" -#include "statedb.hpp" -#include "celldb.hpp" -#include "archive-db.hpp" -#include "archive-manager.hpp" - -#include -#include - -namespace ton { - -namespace validator { - -class ArchiveFileMover : public td::actor::Actor { - public: - ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - void start_up() override; - void got_block_handle0(td::Result R); - void got_block_handle1(td::Result R); - void got_block_handle2(td::Result R); - void got_block_handle(); - - void processed_child(); - void processed_all_children(); - - void got_block_data(td::Result R); - void got_block_proof(td::Result R); - void got_block_proof_link(td::Result R); - - void written_data(); - void written_handle(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - BlockHandle handle_; - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice proof_link_; - bool left_ = true; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveKeyBlockMover : public td::actor::Actor { - public: - ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, - td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - - void start_up() override; - void failed_to_get_proof0(); - void failed_to_get_proof1(); - void failed_to_get_proof2(); - void failed_to_get_proof3(); - void got_block_proof(td::BufferSlice data); - void skip_block_proof(td::BufferSlice data); - - void written_data(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - td::BufferSlice data_; - bool proof_link_ = false; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveMover : public td::actor::Actor { - public: - ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id, - BlockIdExt key_block_id); - - void start_up() override; - void moved_blocks(); - void got_handle(BlockHandle handle); - void got_state(td::Ref state); - void moved_key_blocks(); - void run(); - void completed(); - void add_to_move(BlockIdExt block_id); - void add_to_check(BlockIdExt block_id); - - void got_to_check_handle(td::Result R); - - void abort_query(td::Status error); - void finish_query(); - - private: - std::string db_root_; - BlockHandle handle_; - td::Ref state_; - - td::actor::ActorOwn block_db_; - td::actor::ActorOwn file_db_; - td::actor::ActorOwn old_archive_db_; - td::actor::ActorOwn old_archive_manager_; - td::actor::ActorOwn archive_manager_; - td::actor::ActorOwn cell_db_; - - BlockIdExt masterchain_block_id_; - BlockIdExt shard_block_id_; - BlockIdExt key_block_id_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 41ed50997..50d705a9f 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -554,14 +554,14 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 before_query(); auto value = static_cast(archive_id >> 32); PackageInfo *p; - if (shard_split_depth_ == 0) { - TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); - } else { + if (shard_separated_) { if (value >= packages_.size()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); return; } p = &packages_[value]; + } else { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); @@ -574,10 +574,10 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shar promise.set_result(archive_id_); } else { TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); - if (shard_split_depth_ == 0) { - promise.set_result(p->seqno * (1ull << 32) + archive_id_); - } else { + if (shard_separated_) { promise.set_result(p->idx * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); } } } @@ -609,8 +609,10 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { shard_split_depth_ = td::to_integer(value); CHECK(shard_split_depth_ <= 60); + shard_separated_ = true; } else { shard_split_depth_ = 0; + shard_separated_ = false; } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); @@ -625,16 +627,16 @@ void ArchiveSlice::before_query() { } td::uint32 seqno; ShardIdFull shard_prefix; - if (shard_split_depth_ == 0) { - seqno = archive_id_ + slice_size_ * i; - shard_prefix = ShardIdFull{masterchainId}; - } else { + if (shard_separated_) { R2 = kv_->get(PSTRING() << "info." << i, value); R2.ensure(); CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); unsigned long long shard; CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); shard_prefix.shard = shard; + } else { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; } add_package(seqno, shard_prefix, len, ver); } @@ -651,10 +653,9 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { - kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); - kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); - } + shard_separated_ = true; + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); kv_->commit_transaction().ensure(); add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { @@ -686,6 +687,24 @@ void ArchiveSlice::close_files() { } } +void ArchiveSlice::iterate_block_handles(std::function f) { + before_query(); + td::uint32 range_start = ton_api::db_blockdb_key_value::ID; + td::uint32 range_end = ton_api::db_blockdb_key_value::ID + 1; + kv_->for_each_in_range(td::Slice{(char *)&range_start, 4}, td::Slice{(char *)&range_end, 4}, + [&](td::Slice key, td::Slice value) -> td::Status { + auto r_key = fetch_tl_object(key, true); + if (r_key.is_error()) { + return td::Status::OK(); + } + auto r_handle = create_block_handle(value); + if (r_handle.is_ok()) { + f(*r_handle.ok()); + } + return td::Status::OK(); + }); +} + void ArchiveSlice::do_close() { if (destroyed_) { return; @@ -779,7 +798,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno } masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - if (shard_split_depth_ == 0) { + if (!shard_separated_) { shard_prefix = ShardIdFull{masterchainId}; } else if (!shard_prefix.is_masterchain()) { shard_prefix.shard |= 1; // In case length is < split depth @@ -795,7 +814,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); } commit_transaction(); @@ -1079,7 +1098,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td:: package.idx = i; kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); } id_to_package_[{package.seqno, package.shard_prefix}] = i; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index a027ec0ff..47232252a 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -127,6 +127,8 @@ class ArchiveSlice : public td::actor::Actor { void open_files(); void close_files(); + void iterate_block_handles(std::function f); + private: void before_query(); void do_close(); @@ -159,6 +161,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + bool shard_separated_{false}; td::uint32 shard_split_depth_ = 0; enum Status { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e86a373d1..f4a3a084b 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -27,6 +27,15 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "block/block-auto.h" +#include "permanent-celldb/permanent-celldb-utils.h" +#include "td/actor/MultiPromise.h" + +#include +#include + +#include +#include namespace ton { @@ -73,6 +82,101 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId td::Slice { + return td::Slice(value.data(), value.size()); + } + bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const override { + CHECK(merge_in.existing_value); + auto& value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + vm::CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + vm::CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left, const rocksdb::Slice& right, + std::string* new_value, rocksdb::Logger* logger) const override { + *new_value = left.ToString(); + vm::CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +void CellDbIn::validate_meta() { + LOG(INFO) << "Validating metadata\n"; + size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits::max() : 10000; + auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok(); + bool partial_check = meta.size() == max_meta_keys_loaded; + if (partial_check) { + LOG(ERROR) << "Too much metadata in the database, do only partial check"; + } + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + + // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!partial_check && !root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + LOG(INFO) << "Validating metadata: OK\n"; +} + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -96,44 +200,108 @@ void CellDbIn::start_up() { db_options.snapshot_statistics = snapshot_statistics_; } db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + auto o_celldb_cache_size = opts_->get_celldb_cache_size(); + + std::optional boc_in_memory_options; + std::optional boc_v1_options; + std::optional boc_v2_options; + + if (opts_->get_celldb_v2()) { + boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; + size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); + if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { + LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " + << td::format::as_size(o_celldb_cache_size.value()); + o_celldb_cache_size = min_rocksdb_cache; + } + LOG(WARNING) << "Using V2 DynamicBagOfCells with options " << *boc_v2_options; + } else if (opts_->get_celldb_in_memory()) { + // default options + boc_in_memory_options = vm::DynamicBagOfCellsDb::CreateInMemoryOptions{ + .extra_threads = std::thread::hardware_concurrency(), + .verbose = true, + .use_arena = false, + .use_less_memory_during_creation = true, + }; + LOG(WARNING) << "Using InMemory DynamicBagOfCells with options " << *boc_v2_options; + } else { + boc_v1_options = vm::DynamicBagOfCellsDb::CreateV1Options{}; + LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; + } + + db_options.enable_bloom_filter = !opts_->get_celldb_disable_bloom_filter(); + db_options.two_level_index_and_filter = db_options.enable_bloom_filter + && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days + if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { + o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); + } + + if (o_celldb_cache_size) { + db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // NB: from now on we MUST use this merge operator + // Only V2 and InMemory BoC actually use them, but it still should be kept for V1, + // to handle updates written by V2 or InMemory BoCs + db_options.merge_operator = std::make_shared(); + if (opts_->get_celldb_in_memory()) { td::RocksDbOptions read_db_options; read_db_options.use_direct_reads = true; read_db_options.no_block_cache = true; read_db_options.block_cache = {}; + read_db_options.merge_operator = std::make_shared(); LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; td::Timer timer; auto read_cell_db = std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), *boc_in_memory_options); in_memory_load_time_ = timer.elapsed(); - td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + + // no reads will be allowed from rocksdb, only writes + db_options.no_reads = true; } auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); rocks_db_ = rocks_db->raw_db(); cell_db_ = std::move(rocks_db); if (!opts_->get_celldb_in_memory()) { - boc_ = vm::DynamicBagOfCellsDb::create(); + if (opts_->get_celldb_v2()) { + boc_ = vm::DynamicBagOfCellsDb::create_v2(*boc_v2_options); + } else { + boc_ = vm::DynamicBagOfCellsDb::create(*boc_v1_options); + } boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } + validate_meta(); + alarm_timestamp() = td::Timestamp::in(10.0); auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); + boc_->commit(stor); cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + + if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { + send_closure(parent_, &CellDb::set_thread_safe_boc, boc_); + } else { + send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } if (opts_->get_celldb_preload_all()) { @@ -161,7 +329,7 @@ void CellDbIn::start_up() { { std::string key = "stats.last_deleted_mc_seqno", value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); if (R.ok() == td::KeyValue::GetStatus::Ok) { auto r_value = td::to_integer_safe(value); @@ -169,6 +337,30 @@ void CellDbIn::start_up() { last_deleted_mc_state_ = r_value.move_as_ok(); } } + { + std::string key = "opts.permanent_mode", value; + auto R = boc_->meta_get(td::as_slice(key), value); + R.ensure(); + bool stored_permanent_mode = R.ok() == td::KeyValue::GetStatus::Ok; + permanent_mode_ = stored_permanent_mode || opts_->get_permanent_celldb(); + if (permanent_mode_) { + LOG(WARNING) << "Celldb is in permanent mode"; + if (!stored_permanent_mode) { + cell_db_->begin_write_batch().ensure(); + value = "1"; + vm::CellStorer stor{*cell_db_}; + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + } + cell_db_statistics_.permanent_mode_ = permanent_mode_; + LOG_IF(FATAL, permanent_mode_ && opts_->get_celldb_in_memory()) + << "celldb permanent_mode and in_memory_mode are not compatible"; + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -204,7 +396,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi auto R = get_block(key_hash); // duplicate if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); return; } @@ -215,7 +409,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi cell = std::move(cell)](td::Result Res) mutable { Res.ensure(); timer_prepare.pause(); - td::actor::send_lambda( + td::actor::send_lambda_later( SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { TD_PERF_COUNTER(celldb_store_cell); auto empty = get_empty_key_hash(); @@ -240,10 +434,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::Timer timer_write; vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); set_block(get_empty_key_hash(), std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); timer_write.pause(); @@ -252,7 +446,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); @@ -265,15 +461,159 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi } void CellDbIn::get_cell_db_reader(td::Promise> promise) { + if (db_busy_) { + action_queue_.push([self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); + return; + } + promise.set_result(boc_->get_cell_db_reader()); +} + +void CellDbIn::store_block_state_permanent(td::Ref block, td::Promise> promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } if (db_busy_) { action_queue_.push( - [self = this, promise = std::move(promise)](td::Result R) mutable { + [self = this, block = std::move(block), promise = std::move(promise)](td::Result R) mutable { R.ensure(); - self->get_cell_db_reader(std::move(promise)); + self->store_block_state_permanent(std::move(block), std::move(promise)); }); return; } - promise.set_result(boc_->get_cell_db_reader()); + auto key_hash = get_key_hash(block->block_id()); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + delay_action([cell = boc_->load_cell(R.ok().root_hash.as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); + return; + } + store_block_state_permanent_bulk( + {block}, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise.set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise.set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + RootHash state_root_hash = new_state_root->get_hash(0).bits(); + td::actor::send_closure(SelfId, &CellDbIn::load_cell, state_root_hash, std::move(promise)); + }); +} + +void CellDbIn::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, blocks = std::move(blocks), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent_bulk(std::move(blocks), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecellbulk", 0.1}; + td::Timer timer_prepare; + std::map> new_blocks; + for (auto& block : blocks) { + BlockIdExt block_id = block->block_id(); + if (new_blocks.contains(block_id)) { + continue; + } + if (get_block(get_key_hash(block_id)).is_ok()) { + continue; + } + new_blocks[block_id] = std::move(block); + } + if (new_blocks.empty()) { + promise.set_value(td::Unit{}); + return; + } + for (auto& [block_id, block] : new_blocks) { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS_PROMISE(promise, + block::unpack_block_prev_blk_try(block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt& prev_id : prev) { + if (!new_blocks.contains(prev_id) && get_block(get_key_hash(prev_id)).is_error()) { + promise.set_error(td::Status::Error("cannot store block state: previous block is not in db")); + return; + } + } + } + db_busy_ = true; + calculate_permanent_celldb_update( + new_blocks, async_executor, + [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), + promise = std::move(promise)](td::Result> R) mutable { + td::actor::send_lambda_later( + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), R = std::move(R), + promise = std::move(promise)]() mutable { + SCOPE_EXIT { + release_db(); + }; + TRY_RESULT_PROMISE(promise, updates, std::move(R)); + TD_PERF_COUNTER(celldb_store_cell_multi); + timer_prepare.pause(); + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + + for (auto& update : updates) { + for (auto& [k, v] : update.to_store) { + cell_db_->set(k.as_slice(), v).ensure(); + } + } + + CHECK(!updates.empty()); + auto empty = get_empty_key_hash(); + auto E = get_block(empty).move_as_ok(); + for (size_t i = 0; i < updates.size(); ++i) { + KeyHash prev = i == 0 ? empty : get_key_hash(updates[i - 1].block_id); + KeyHash next = i + 1 == updates.size() ? E.next : get_key_hash(updates[i + 1].block_id); + DbEntry entry{updates[i].block_id, prev, next, updates[i].state_root_hash}; + set_block(get_key_hash(updates[i].block_id), std::move(entry)); + } + E.next = get_key_hash(updates[0].block_id); + if (E.prev == empty) { + E.prev = get_key_hash(updates.back().block_id); + } + set_block(empty, std::move(E)); + + boc_->commit(stor).ensure(); // Save meta + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + cell_db_statistics_.store_cell_bulk_queries_++; + cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); + } + promise.set_result(td::Unit()); + }); + }); } std::vector> CellDbIn::prepare_stats() { @@ -343,6 +683,7 @@ void CellDbIn::flush_db_stats() { } td::RocksDb::reset_statistics(statistics_); cell_db_statistics_.clear(); + cell_db_statistics_.permanent_mode_ = permanent_mode_; } void CellDbIn::alarm() { @@ -361,6 +702,10 @@ void CellDbIn::alarm() { << " queue_size=" << cells_to_migrate_.size(); migration_stats_ = {}; } + if (permanent_mode_) { + skip_gc(); + return; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -414,6 +759,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { }); return; } + CHECK(!permanent_mode_); td::PerfWarningTimer timer{"gccell", 0.1}; td::PerfWarningTimer timer_all{"gccell_all", 0.05}; @@ -440,16 +786,20 @@ void CellDbIn::gc_cont2(BlockHandle handle) { timer_get_keys.reset(); td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + auto r_cell = boc_->load_cell(F.root_hash.as_slice()); + td::Ref cell; + if (r_cell.is_ok()) { + cell = r_cell.move_as_ok(); + boc_->dec(cell); + } - boc_->dec(cell); db_busy_ = true; boc_->prepare_commit_async( async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { TD_PERF_COUNTER(celldb_gc_cell); @@ -458,17 +808,19 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(key_hash)).ensure(); + boc_->meta_erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); if (handle->id().is_masterchain()) { last_deleted_mc_state_ = handle->id().seqno(); std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - cell_db_->set(td::as_slice(key), td::as_slice(value)); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); } + + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -530,7 +882,7 @@ CellDbIn::KeyHash CellDbIn::get_empty_key_hash() { td::Result CellDbIn::get_block(KeyHash key_hash) { const auto key = get_key(key_hash); std::string value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); auto S = R.move_as_ok(); if (S == td::KeyValue::GetStatus::NotFound) { @@ -543,10 +895,13 @@ td::Result CellDbIn::get_block(KeyHash key_hash) { void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { const auto key = get_key(key_hash); - cell_db_->set(td::as_slice(key), e.release()).ensure(); + boc_->meta_set(td::as_slice(key), e.release()); } void CellDbIn::migrate_cell(td::Bits256 hash) { + if (permanent_mode_) { + return; + } cells_to_migrate_.insert(hash); if (!migration_active_) { migration_active_ = true; @@ -631,12 +986,13 @@ void CellDb::alarm() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - if (in_memory_boc_) { - auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (thread_safe_boc_) { + auto result = thread_safe_boc_->load_root_thread_safe(hash.as_slice()); if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { - LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + return; } } if (!started_) { @@ -658,6 +1014,14 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::store_block_state_permanent(td::Ref block, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent, std::move(block), std::move(promise)); +} + +void CellDb::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void CellDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } @@ -694,9 +1058,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { std::vector> CellDbIn::CellDbStatistics::prepare_stats() { std::vector> stats; + stats.emplace_back("permanent_mode", PSTRING() << permanent_mode_); stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + if (permanent_mode_) { + stats.emplace_back("store_cell.bulk.queries", PSTRING() << store_cell_bulk_queries_); + stats.emplace_back("store_cell.bulk.total_blocks", PSTRING() << store_cell_bulk_total_blocks_); + } stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); @@ -710,6 +1079,13 @@ std::vector> CellDbIn::CellDbStatistics::pre for (auto& [key, value] : boc_stats_->custom_stats) { stats.emplace_back(key, value); } + + for (auto& [key, value] : boc_stats_->named_stats.stats_str) { + stats.emplace_back(key, value); + } + for (auto& [key, value] : boc_stats_->named_stats.stats_int) { + stats.emplace_back(key, td::to_string(value)); + } } return stats; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 5639b9748..f70f6aef8 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -66,6 +66,8 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -74,6 +76,7 @@ class CellDbIn : public CellDbBase { CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, td::Ref opts); + void validate_meta(); void start_up() override; void alarm() override; @@ -136,9 +139,12 @@ class CellDbIn : public CellDbBase { std::unique_ptr migration_stats_; struct CellDbStatistics { + bool permanent_mode_; PercentileStats store_cell_time_; PercentileStats store_cell_prepare_time_; PercentileStats store_cell_write_time_; + size_t store_cell_bulk_queries_ = 0; + size_t store_cell_bulk_total_blocks_ = 0; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); std::optional in_memory_load_time_; @@ -155,6 +161,7 @@ class CellDbIn : public CellDbBase { CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); BlockSeqno last_deleted_mc_state_ = 0; + bool permanent_mode_ = false; bool db_busy_ = false; std::queue> action_queue_; @@ -187,6 +194,8 @@ class CellDb : public CellDbBase { void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void update_snapshot(std::unique_ptr snapshot) { CHECK(!opts_->get_celldb_in_memory()); if (!started_) { @@ -195,13 +204,13 @@ class CellDb : public CellDbBase { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } - void set_in_memory_boc(std::shared_ptr in_memory_boc) { - CHECK(opts_->get_celldb_in_memory()); + void set_thread_safe_boc(std::shared_ptr thread_safe_boc) { + CHECK(opts_->get_celldb_in_memory() || opts_->get_celldb_v2()); if (!started_) { alarm(); } started_ = true; - in_memory_boc_ = std::move(in_memory_boc); + thread_safe_boc_ = std::move(thread_safe_boc); } void get_cell_db_reader(td::Promise> promise); @@ -219,7 +228,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; - std::shared_ptr in_memory_boc_; + std::shared_ptr thread_safe_boc_; bool started_ = false; std::vector> prepared_stats_{{"started", "false"}}; diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index feefd1f9c..4e87327dc 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -106,6 +106,16 @@ std::string ZeroStateShort::filename_short() const { return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex(); } +std::string SplitAccountState::filename_short() const { + return PSTRING() << "stateaccount_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << shard_to_str(effective_shard) << "_" << hash().to_hex(); +} + +std::string SplitPersistentState::filename_short() const { + return PSTRING() << "statesplit_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << hash().to_hex(); +} + PersistentStateShort PersistentState::shortref() const { return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()}; } @@ -258,37 +268,6 @@ std::string BlockInfoShort::filename_short() const { } // namespace fileref -FileReference::FileReference(tl_object_ptr key) { - ton_api::downcast_call( - *key.get(), - td::overloaded( - [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; }, - [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_zeroStateFile& key) { - ref_ = fileref::ZeroState{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_persistentStateFile& key) { - ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)}; - }, - [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_proofLink& key) { - ref_ = fileref::ProofLink{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_signatures& key) { - ref_ = fileref::Signatures{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_candidate& key) { - ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), - key.id_->collated_data_file_hash_}; - }, - [&](const ton_api::db_filedb_key_candidateRef& key) { - ref_ = fileref::CandidateRef{create_block_id(key.id_)}; - }, - [&](const ton_api::db_filedb_key_blockInfo& key) { - ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; - })); -} - FileReferenceShort FileReference::shortref() const { FileReferenceShort h; ref_.visit([&](const auto& obj) { h = obj.shortref(); }); @@ -319,6 +298,24 @@ ShardIdFull FileReferenceShort::shard() const { return h; } +BlockSeqno FileReferenceShort::seqno_of_persistent_state() const { + BlockSeqno result; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitAccountState& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitPersistentState& x) { result = x.masterchain_seqno; }, + [&](const fileref::ZeroStateShort) { result = 0; }, [&](const auto&) { CHECK(false); })); + return result; +} + +bool FileReferenceShort::is_state_like() const { + bool result = false; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = true; }, + [&](const fileref::SplitAccountState& x) { result = true; }, + [&](const fileref::SplitPersistentState& x) { result = true; }, + [&](const fileref::ZeroStateShort) { result = true; }, [&](const auto&) {})); + return result; +} + std::string FileReference::filename() const { std::string h; ref_.visit([&](const auto& obj) { h = obj.filename(); }); @@ -451,6 +448,35 @@ td::Result FileReferenceShort::create(std::string filename) } else { return td::Status::Error(ErrorCode::protoviolation, "too big file name"); } + } else if (token == "stateaccount") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(effective_shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + if (!shard_is_proper_ancestor(shard, effective_shard)) { + return td::Status::Error(ErrorCode::protoviolation, "shard is not a proper ancestor of effective shard"); + } + return fileref::SplitAccountState{ShardIdFull{workchain, shard}, effective_shard, masterchain_seqno, vhash}; + } else if (token == "statesplit") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + return fileref::SplitPersistentState{ShardIdFull{workchain, shard}, masterchain_seqno, vhash}; } else if (token == "state") { std::getline(ss, token, '_'); TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 6710cc063..df368f759 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -110,8 +110,77 @@ class ZeroState { BlockIdExt block_id; }; +class SplitAccountState { + public: + static SplitAccountState create(BlockIdExt block_id, BlockIdExt masterchain_block_id, ShardId effective_shard) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id), effective_shard); + + return { + .shard_id = block_id.shard_full(), + .effective_shard = effective_shard, + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return {shard_id.workchain, effective_shard}; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + ShardId effective_shard; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + +class SplitPersistentState { + public: + static SplitPersistentState create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return shard_id; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + class PersistentStateShort { public: + static PersistentStateShort create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + FileHash hash() const { return hashv; } @@ -346,9 +415,10 @@ class BlockInfo { class FileReferenceShort { private: - td::Variant + td::Variant ref_; public: @@ -366,6 +436,8 @@ class FileReferenceShort { td::Bits256 hash() const; ShardIdFull shard() const; + BlockSeqno seqno_of_persistent_state() const; + bool is_state_like() const; std::string filename_short() const; }; @@ -382,7 +454,6 @@ class FileReference { } FileReference() : ref_(fileref::Empty{}) { } - FileReference(tl_object_ptr key); static td::Result create(std::string filename); diff --git a/validator/db/package.cpp b/validator/db/package.cpp index cc699baa8..be9616422 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -48,8 +48,16 @@ Package::Package(td::FileFd fd) : fd_(std::move(fd)) { } td::Status Package::truncate(td::uint64 size) { - TRY_STATUS(fd_.seek(size + header_size())); - return fd_.truncate_to_current_position(size + header_size()); + auto target_size = size + header_size(); + TRY_RESULT(current_size, fd_.get_size()); + + // Only truncate if the size actually differs to avoid updating mtime unnecessarily + if (current_size != target_size) { + TRY_STATUS(fd_.seek(target_size)); + return fd_.truncate_to_current_position(target_size); + } + + return td::Status::OK(); } td::uint64 Package::append(std::string filename, td::Slice data, bool sync) { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp new file mode 100644 index 000000000..b1d19ceb4 --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "permanent-celldb-utils.h" +#include "block/block-auto.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/HashMap.h" +#include "vm/db/CellStorage.h" + +namespace ton::validator { + +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + auto updates = std::make_shared>(); + updates->reserve(blocks.size()); + for (auto& [_, block] : blocks) { + executor->execute_async([block = block, updates, executor, + promise = std::make_shared>(ig.get_promise())]() mutable { + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise->set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise->set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + td::HashMap visited; + PermanentCellDbUpdate update{.block_id = block->block_id(), + .state_root_hash = new_state_root->get_hash(0).bits()}; + std::function&, int)> dfs = [&](const td::Ref& cell, int merkle_depth) { + int& vis = visited[cell->get_hash()]; + if (vis & (1 << merkle_depth)) { + return; + } + vis |= (1 << merkle_depth); + vm::CellSlice cs{vm::NoVm(), cell}; + if (cs.special_type() == vm::CellTraits::SpecialType::PrunnedBranch && cell->get_level() == merkle_depth + 1) { + return; + } + update.to_store.emplace_back( + cell->get_hash(merkle_depth), + vm::CellStorer::serialize_value(1 << 29, cell->load_cell().move_as_ok().data_cell, false, merkle_depth)); + merkle_depth = cs.child_merkle_depth(merkle_depth); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), merkle_depth); + } + }; + dfs(new_state_root, 0); + executor->execute_sync( + [update = std::move(update), updates = std::move(updates), promise = std::move(promise)]() mutable { + updates->push_back(std::move(update)); + promise->set_result(td::Unit()); + }); + }); + } + + ig.add_promise([updates, promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + promise.set_value(std::move(*updates)); + }); +} + +} // namespace ton::validator diff --git a/adnl/adnl-static-nodes.h b/validator/db/permanent-celldb/permanent-celldb-utils.h similarity index 54% rename from adnl/adnl-static-nodes.h rename to validator/db/permanent-celldb/permanent-celldb-utils.h index 91e3bed0c..fad2ddfcd 100644 --- a/adnl/adnl-static-nodes.h +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -13,31 +13,25 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "interfaces/block.h" +#include "vm/db/DynamicBagOfCellsDb.h" -#include "td/actor/actor.h" -#include "td/utils/Status.h" -#include "td/actor/PromiseFuture.h" -#include "auto/tl/ton_api.h" - -#include "adnl-peer-table.h" - -namespace ton { +#include +#include -namespace adnl { +namespace ton::validator { -class AdnlStaticNodesManager : public td::actor::Actor { - public: - virtual void add_node(AdnlNode node) = 0; - virtual void del_node(AdnlNodeIdShort id) = 0; - virtual td::Result get_node(AdnlNodeIdShort id) = 0; - - static td::actor::ActorOwn create(); +struct PermanentCellDbUpdate { + BlockIdExt block_id; + RootHash state_root_hash; + std::vector> to_store; }; +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise); -} // namespace adnl - -} // namespace ton +} // namespace ton::validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d..636ca4200 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,6 +259,41 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +void RootDb::store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + if (handle->id() != block->block_id()) { + promise.set_error(td::Status::Error("block id mismatch")); + return; + } + if (handle->moved_to_archive() || handle->inited_state_boc()) { + get_block_state(handle, std::move(promise)); + return; + } + auto P = td::PromiseCreator::lambda( + [b = archive_db_.get(), handle, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, root, std::move(R)); + handle->set_state_root_hash(root->get_hash().bits()); + handle->set_state_boc(); + + auto S = create_shard_state(handle->id(), std::move(root)); + S.ensure(); + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), state = S.move_as_ok()](td::Result R) mutable { + R.ensure(); + promise.set_value(std::move(state)); + }); + + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); + }); + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent, std::move(block), std::move(P)); +} + +void RootDb::store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { if (handle->inited_state_boc()) { if (handle->deleted_state_boc()) { @@ -281,39 +316,46 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise cell, + td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDb::store_cell, BlockIdExt{effective_block}, cell, std::move(promise)); +} + void RootDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } -void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, +void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } -void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, +void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, type, std::move(promise)); } -void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { +void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id, - offset, max_size, std::move(promise)); + type, offset, max_size, std::move(promise)); } -void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, - std::move(promise)); +void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, + type, std::move(promise)); } void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { @@ -431,10 +473,6 @@ void RootDb::allow_state_gc(BlockIdExt block_id, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_state_gc, block_id, std::move(promise)); } -void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise)); -} - void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); @@ -519,7 +557,7 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } -void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { +void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl); } @@ -531,6 +569,10 @@ void RootDb::get_persistent_state_descriptions(td::Promise f) { + td::actor::send_closure(archive_db_, &ArchiveManager::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4..14c30f6ee 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -62,7 +62,13 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_block_handle(BlockHandle handle, td::Promise promise) override; @@ -71,17 +77,18 @@ class RootDb : public Db { td::actor::send_closure(validator_manager_, &ValidatorManager::get_block_handle, id, force, std::move(promise)); } - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, std::function write_data, td::Promise promise) override; - void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; - void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; @@ -118,8 +125,6 @@ class RootDb : public Db { void archive(BlockHandle handle, td::Promise promise) override; void allow_state_gc(BlockIdExt block_id, td::Promise promise); - void allow_block_gc(BlockIdExt block_id, td::Promise promise); - //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise); void prepare_stats(td::Promise>> promise) override; @@ -137,10 +142,11 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override; + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) override; void add_persistent_state_description(td::Ref desc, td::Promise promise) override; void get_persistent_state_descriptions(td::Promise>> promise) override; + void iterate_temp_block_handles(std::function f) override; private: td::actor::ActorId validator_manager_; diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index fe3b9d735..753ed7e64 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -277,16 +277,36 @@ void StateDb::add_persistent_state_description(td::Reflist_.resize(new_size); - std::vector> shard_blocks; - for (const BlockIdExt& block_id : desc->shard_blocks) { - shard_blocks.push_back(create_tl_block_id(block_id)); + bool can_be_stored_as_v1 = true; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + if (split_depth != 0) { + can_be_stored_as_v1 = false; + break; + } } + + td::BufferSlice serialized_shards; + + if (can_be_stored_as_v1) { + std::vector> shard_blocks; + for (auto const& [block_id, _] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } else { + std::vector> shard_blocks; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_object( + create_tl_block_id(block_id), static_cast(split_depth))); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } + auto key = create_hash_tl_object(desc->masterchain_id.seqno()); - kv_->set(key.as_slice(), - create_serialize_tl_object(std::move(shard_blocks)) - .as_slice()) - .ensure(); + kv_->set(key.as_slice(), serialized_shards.as_slice()).ensure(); list->list_.push_back(create_tl_object( create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); @@ -325,11 +345,26 @@ void StateDb::get_persistent_state_descriptions(td::Promise(value, true); + auto F2 = fetch_tl_object(value, true); F2.ensure(); - for (const auto& block_id : F2.ok()->shard_blocks_) { - desc.shard_blocks.push_back(create_block_id(block_id)); - } + ton_api::downcast_call(*F2.ok().get(), + td::overloaded( + [&](ton_api::db_state_persistentStateDescriptionShards const& shards) { + for (auto const& block : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(block), + .split_depth = 0, + }); + } + }, + [&](ton_api::db_state_persistentStateDescriptionShardsV2 const& shards) { + for (auto const& shard_description : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(shard_description->block_), + .split_depth = static_cast(shard_description->split_depth_), + }); + } + })); result.push_back(td::Ref(true, std::move(desc))); } promise.set_result(std::move(result)); diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8473cb228..d60811d59 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -21,22 +21,126 @@ #include "common/checksum.h" #include "common/delay.h" #include "ton/ton-io.hpp" +#include "vm/cells/MerkleProof.h" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" namespace ton { namespace validator { -DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise> promise) +class SplitStateDeserializer { + public: + td::Result> get_effective_shards_from_header(ShardId shard_id, RootHash root_hash, + td::Ref wrapped_header, + td::uint32 split_depth) { + int shard_prefix_length = shard_pfx_len(shard_id); + CHECK(split_depth <= 63 && shard_prefix_length < static_cast(split_depth)); + + try { + TRY_RESULT(header, vm::MerkleProof::try_virtualize(wrapped_header)); + + if (RootHash{header->get_hash().bits()} != root_hash) { + return td::Status::Error("Hash mismatch in split state header"); + } + + auto shard_state_cs = vm::load_cell_slice(header); + bool rc = block::gen::t_ShardStateUnsplit.unpack(shard_state_cs, shard_state_); + if (!rc) { + return td::Status::Error("Cannot deserialize ShardStateUnsplit"); + } + + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(shard_state_.accounts), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + std::vector parts; + + // The following loop is the same as in state-serializer.cpp. + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + parts.push_back({effective_shard, account_dict_part.get_wrapped_dict_root()->get_hash()}); + } + } + + // Now check that header does not contain pruned cells outside of accounts dict. For that, we + // just replace account dict with an empty cell and see if header remains virtualized or not. + shard_state_.accounts = vm::DataCell::create("", 0, {}, false).move_as_ok(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + if (cb.finalize()->get_virtualization() > 0) { + return td::Status::Error("State headers is pruned outside of account dict"); + } + + return parts; + } catch (vm::VmVirtError const&) { + return td::Status::Error("Insufficient number of cells in split state header"); + } + } + + td::Ref merge(std::vector> const& parts) { + vm::AugmentedDictionary accounts{256, block::tlb::aug_ShardAccounts}; + for (auto const& part_root : parts) { + vm::AugmentedDictionary part{ + vm::load_cell_slice_ref(part_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + bool rc = accounts.combine_with(part); + LOG_CHECK(rc) << "Split state parts have been validated but merging them still resulted in a conflict"; + } + + CHECK(accounts.is_valid()); + + shard_state_.accounts = accounts.get_wrapped_dict_root(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + auto state_root = cb.finalize(); + CHECK(state_root->get_virtualization() == 0); + return state_root; + } + + private: + block::gen::ShardStateUnsplit::Record shard_state_; +}; + +DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, + td::uint32 priority, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise> promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , split_depth_(split_depth) , priority_(priority) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || split_depth_ == 0); + + int shard_prefix_length = shard_pfx_len(block_id_.shard_full().shard); + if (shard_prefix_length >= static_cast(split_depth_)) { + split_depth_ = 0; + } + + LOG(INFO) << "requested to download state of " << block_id.to_str() << " referenced by " + << masterchain_block_id.to_str() << " with split depth " << split_depth; } +DownloadShardState::~DownloadShardState() = default; + void DownloadShardState::start_up() { status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; @@ -64,6 +168,8 @@ void DownloadShardState::got_block_handle(BlockHandle handle) { } void DownloadShardState::retry() { + deserializer_ = {}; + parts_.clear(); download_state(); } @@ -72,7 +178,22 @@ void DownloadShardState::download_state() { checked_proof_link(); return; } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Cannot get proof link from import: " << R.move_as_error(); + td::actor::send_closure(SelfId, &DownloadShardState::download_proof_link); + } else { + LOG(INFO) << "Got proof link for " << block_id.to_str() << " from import"; + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_proof_link, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, block_id_, + masterchain_block_id_, std::move(P)); +} + +void DownloadShardState::download_proof_link() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { fail_handler(SelfId, R.move_as_error()); @@ -82,7 +203,6 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { @@ -112,20 +232,35 @@ void DownloadShardState::checked_proof_link() { } }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, block_id_.file_hash, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading zero state"); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - fail_handler(SelfId, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); - } - }); CHECK(masterchain_block_id_.is_valid()); CHECK(masterchain_block_id_.is_masterchain()); - td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, - masterchain_block_id_, priority_, std::move(P)); + + if (split_depth_ == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_split_state_header, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitPersistentStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state header"); + } } - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -189,10 +324,137 @@ void DownloadShardState::checked_shard_state() { std::move(P)); } else { td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, - std::move(data_), std::move(P)); + UnsplitStateType{}, std::move(data_), std::move(P)); } } +void DownloadShardState::downloaded_split_state_header(td::BufferSlice data) { + LOG(INFO) << "processing state header"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state header"); + + deserializer_ = std::make_unique(); + + auto maybe_header = vm::std_boc_deserialize(data); + if (maybe_header.is_error()) { + fail_handler(actor_id(this), maybe_header.move_as_error()); + return; + } + + auto maybe_parts = deserializer_->get_effective_shards_from_header(block_id_.shard_full().shard, handle_->state(), + maybe_header.move_as_ok(), split_depth_); + if (maybe_parts.is_error()) { + fail_handler(actor_id(this), maybe_parts.move_as_error()); + return; + } + + parts_ = maybe_parts.move_as_ok(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitPersistentStateType{}, std::move(data), std::move(P)); +} + +namespace { + +void retry_part_download(td::actor::ActorId SelfId, td::Status error) { + LOG(WARNING) << "failed to download state part : " << error; + delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); }, + td::Timestamp::in(1.0)); +} + +} // namespace + +void DownloadShardState::download_next_part_or_finish() { + if (stored_parts_.size() == parts_.size()) { + auto state_root = deserializer_->merge(stored_parts_); + auto maybe_state = create_shard_state(block_id_, state_root); + + // We cannot rollback database changes here without significant elbow grease. + maybe_state.ensure(); + state_ = maybe_state.move_as_ok(); + CHECK(state_->root_hash() == handle_->state()); + + written_shard_state_file(); + return; + } + + size_t idx = stored_parts_.size(); + + LOG(INFO) << "downloading state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + retry_part_download(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_state_part, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitAccountStateType{parts_[idx].effective_shard}, priority_, + std::move(P)); +} + +void DownloadShardState::downloaded_state_part(td::BufferSlice data) { + size_t idx = stored_parts_.size(); + + LOG(INFO) << "processing state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto maybe_part = vm::std_boc_deserialize(data); + if (maybe_part.is_error()) { + retry_part_download(actor_id(this), maybe_part.move_as_error()); + return; + } + + auto root = maybe_part.move_as_ok(); + if (root->get_hash() != parts_[idx].root_hash) { + auto error_message = + "Hash mismatch for part " + + persistent_state_type_to_string(block_id_.shard_full(), SplitAccountStateType{parts_[idx].effective_shard}); + retry_part_download(actor_id(this), td::Status::Error(error_message)); + return; + } + + stored_parts_.push_back(root); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_state_part_file); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitAccountStateType{parts_[idx].effective_shard}, std::move(data), std::move(P)); + + LOG(INFO) << "storing state part to file " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state part to file (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + +void DownloadShardState::written_state_part_file() { + size_t idx = stored_parts_.size() - 1; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::saved_state_part_into_celldb, R.move_as_ok()); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_block_state_part, + BlockId{block_id_.shard_full().workchain, parts_[idx].effective_shard, block_id_.seqno()}, + stored_parts_.back(), std::move(P)); + LOG(INFO) << "saving to celldb state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : saving state part to celldb (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + +void DownloadShardState::saved_state_part_into_celldb(td::Ref cell) { + stored_parts_.back() = cell; + download_next_part_or_finish(); +} + void DownloadShardState::written_shard_state_file() { status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index bde80aae1..1c1be04c4 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -25,11 +25,19 @@ namespace ton { namespace validator { +class SplitStateDeserializer; + +struct SplitStatePart { + ShardId effective_shard; + vm::CellHash root_hash; +}; + class DownloadShardState : public td::actor::Actor { public: - DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, + DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise); + ~DownloadShardState(); void start_up() override; void got_block_handle(BlockHandle handle); @@ -38,13 +46,20 @@ class DownloadShardState : public td::actor::Actor { void checked_proof_link(); void download_state(); + void download_proof_link(); void download_zero_state(); void downloaded_zero_state(td::BufferSlice data); void downloaded_shard_state(td::BufferSlice data); - void checked_shard_state(); + + void downloaded_split_state_header(td::BufferSlice data); + void download_next_part_or_finish(); + void downloaded_state_part(td::BufferSlice data); + void written_state_part_file(); + void saved_state_part_into_celldb(td::Ref cell); + void written_shard_state_file(); void written_shard_state(td::Ref state); void written_block_handle(); @@ -58,6 +73,7 @@ class DownloadShardState : public td::actor::Actor { private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + td::uint32 split_depth_; BlockHandle handle_; td::uint32 priority_; @@ -66,6 +82,10 @@ class DownloadShardState : public td::actor::Actor { td::Timestamp timeout_; td::Promise> promise_; + std::unique_ptr deserializer_; + std::vector parts_; + std::vector> stored_parts_; + td::BufferSlice data_; td::Ref state_; diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 53a3d351b..86e8f4b8c 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,7 +106,7 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); - } else if (try_get_candidate_) { + } else if (try_get_candidate_ && !handle_->id().is_masterchain()) { try_get_candidate_ = false; td::actor::send_closure( manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), diff --git a/validator/downloaders/wait-block-state-merge.cpp b/validator/downloaders/wait-block-state-merge.cpp index 2b1961610..c5f1cfa7f 100644 --- a/validator/downloaders/wait-block-state-merge.cpp +++ b/validator/downloaders/wait-block-state-merge.cpp @@ -55,7 +55,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, false, std::move(P_l)); auto P_r = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -66,7 +66,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, false, std::move(P_r)); } diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index c80e7d896..a69e2b130 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -32,7 +32,11 @@ void WaitBlockState::alarm() { } void WaitBlockState::abort_query(td::Status reason) { - if (promise_) { + if (promise_no_store_) { + promise_no_store_.set_error( + reason.clone().move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + } + if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { LOG(WARNING) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; @@ -40,18 +44,19 @@ void WaitBlockState::abort_query(td::Status reason) { LOG(DEBUG) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; } - promise_.set_error(reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + promise_final_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } stop(); } void WaitBlockState::finish_query() { CHECK(handle_->received_state()); - /*if (handle_->id().is_masterchain() && handle_->inited_proof()) { - td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, prev_state_, [](td::Unit) {}); - }*/ - if (promise_) { - promise_.set_result(prev_state_); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + } + if (promise_final_) { + promise_final_.set_result(prev_state_); } stop(); } @@ -68,6 +73,8 @@ void WaitBlockState::start() { return; } bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + bool allow_download = + last_masterchain_state_.is_null() || opts_->need_monitor(handle_->id().shard_full(), last_masterchain_state_); if (handle_->received_state() && inited_proof) { reading_from_db_ = true; @@ -81,8 +88,8 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); - // id.file_hash contrains correct file hash of zero state - // => if file with this sha256 is found it is garanteed to be correct + // id.file_hash contains correct file hash of zero state + // => if file with this sha256 is found it is guaranteed to be correct // => if it is not, this error is permanent auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = handle_->id()](td::Result R) { if (R.is_error()) { @@ -108,7 +115,7 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (check_persistent_state_desc() && !handle_->received_state()) { + } else if (check_persistent_state_desc() && !handle_->received_state() && allow_download) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); @@ -117,12 +124,32 @@ void WaitBlockState::start() { td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); } }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; - td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, - timeout_, std::move(P)) - .release(); + td::uint32 split_depth = 0; + bool block_found = false; + for (auto const& [block, block_split_depth] : persistent_state_desc_->shard_blocks) { + if (block == handle_->id()) { + split_depth = block_split_depth; + block_found = true; + break; + } + } + if (!block_found) { + LOG(ERROR) << "invalid persistent state description passed to WaitBlockState for block " + << handle_->id().to_str(); + P.set_error(td::Status::Error("invalid persistent state description")); + } else { + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, split_depth, + priority_, manager_, timeout_, std::move(P)) + .release(); + } } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, td::Timestamp::in(0.1)); @@ -148,6 +175,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, @@ -161,6 +192,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { + if (!allow_download && !handle_->received()) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, @@ -245,14 +280,6 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { - TD_PERF_COUNTER(apply_block_to_state); - td::PerfWarningTimer t{"applyblocktostate", 0.1}; - auto S = prev_state_.write().apply_block(handle_->id(), block_); - if (S.is_error()) { - abort_query(S.move_as_error_prefix("apply error: ")); - return; - } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); @@ -261,7 +288,23 @@ void WaitBlockState::apply() { } }); + if (opts_->get_permanent_celldb()) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, std::move(P)); + return; + } + TD_PERF_COUNTER(apply_block_to_state); + td::PerfWarningTimer t{"applyblocktostate", 0.1}; + auto S = prev_state_.write().apply_block(handle_->id(), block_); + if (S.is_error()) { + abort_query(S.move_as_error_prefix("apply error: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } void WaitBlockState::written_state(td::Ref upd_state) { @@ -281,6 +324,10 @@ void WaitBlockState::got_state_from_db(td::Ref state) { }); td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } else { finish_query(); } diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 6a14d909f..cf556e33d 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -26,18 +26,23 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: - WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise, + WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, + td::Ref last_masterchain_state, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise> promise_no_store, + td::Promise> promise_final, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) + , opts_(opts) + , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) + , promise_no_store_(std::move(promise_no_store)) + , promise_final_(std::move(promise_final)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -89,9 +94,12 @@ class WaitBlockState : public td::actor::Actor { td::uint32 priority_; + td::Ref opts_; + td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; - td::Promise> promise_; + td::Promise> promise_no_store_; + td::Promise> promise_final_; td::Ref persistent_state_desc_; td::Ref prev_state_; diff --git a/validator/fabric.h b/validator/fabric.h index 2c39acebe..8a3f6193f 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,7 +26,33 @@ namespace ton { namespace validator { -enum CollateMode { skip_store_candidate = 1 }; +struct CollateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + bool is_hardfork = false; + Ed25519_PublicKey creator{td::Bits256::zero()}; + td::Ref validator_set = {}; + td::Ref collator_opts = {}; + adnl::AdnlNodeIdShort collator_node_id = adnl::AdnlNodeIdShort::zero(); + bool skip_store_candidate = false; + int attempt_idx = 0; + + // Optional - used for optimistic collation + Ref optimistic_prev_block = {}; +}; + +struct ValidateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + td::Ref validator_set = {}; + PublicKeyHash local_validator_id = PublicKeyHash::zero();; + bool is_fake = false; + + // Optional - used for validation of optimistic candidates + Ref optimistic_prev_block = {}; +}; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); @@ -39,7 +65,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); @@ -56,13 +82,17 @@ void run_check_external_message(td::Ref message, td::actor::ActorId< void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, + td::Ref approve_signatures, int send_broadcast_mode, bool apply, td::actor::ActorId manager, td::Promise promise); void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); +void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, bool send_block_broadcast, + td::actor::ActorId manager, td::Promise promise); void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); @@ -76,18 +106,10 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp new file mode 100644 index 000000000..e9927518a --- /dev/null +++ b/validator/full-node-fast-sync-overlays.cpp @@ -0,0 +1,580 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "full-node-fast-sync-overlays.hpp" + +#include "checksum.h" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" +#include "full-node-serializer.hpp" + +namespace ton::validator::fullnode { + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in fast sync overlay from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + if (src == local_id_.pubkey_hash()) { + return; // dropping broadcast from self + } + BlockIdExt block_id = create_block_id(query.block_); + ShardIdFull shard_id = create_shard_id(query.dst_shard_); + if (query.proof_->get_id() != ton_api::tonNode_outMsgQueueProof::ID) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with proof not tonNode.outMsgQueueProof"; + return; + } + auto tl_proof = move_tl_object_as(query.proof_); + auto R = OutMsgQueueProof::fetch(shard_id, {block_id}, + block::ImportedMsgQueueLimits{.max_bytes = td::uint32(query.limits_->max_bytes_), + .max_msgs = td::uint32(query.limits_->max_msgs_)}, + *tl_proof); + if (R.is_error()) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proof: " << R.error(); + return; + } + if (R.ok().size() != 1) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proofs count=" << R.ok().size(); + return; + } + auto proof = std::move(R.move_as_ok()[0]); + + LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast to " << shard_id.to_str() << " from " << block_id.to_str() + << ", msgs=" << proof->msg_count_ << ", size=" << tl_proof->queue_proofs_.size(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::add_out_msg_queue_proof, shard_id, + std::move(proof)); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in fast sync overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodeFastSyncOverlay::process_telemetry_broadcast( + adnl::AdnlNodeIdShort src, const tl_object_ptr &telemetry) { + if (telemetry->adnl_id_ != src.bits256_value()) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; + return; + } + auto now = (td::int32)td::Clocks::system(); + if (telemetry->timestamp_ < now - 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" + << now - telemetry->timestamp_ << "s ago)"; + return; + } + if (telemetry->timestamp_ > now + 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" + << telemetry->timestamp_ - now << "s in the future)"; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; + auto s = td::json_encode(td::ToJson(*telemetry), false); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); + telemetry_file_ << s << "\n"; + telemetry_file_.flush(); + if (telemetry_file_.fail()) { + VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; + } +} + +void FullNodeFastSyncOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + if (collect_telemetry_ && src != local_id_.pubkey_hash()) { + auto R = fetch_tl_prefix(broadcast, true); + if (R.is_ok()) { + process_telemetry_broadcast(adnl::AdnlNodeIdShort{src}, R.ok()); + } + } + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodeFastSyncOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in fast sync overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodeFastSyncOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in fast sync overlay (with compression): " + << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in fast sync overlay (with compression): " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::send_validator_telemetry(tl_object_ptr telemetry) { + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_, telemetry); + } + auto data = serialize_tl_object(telemetry, true); + if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } +} + +void FullNodeFastSyncOverlay::collect_validator_telemetry(std::string filename) { + if (collect_telemetry_) { + telemetry_file_.close(); + } + collect_telemetry_ = true; + LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; + telemetry_file_.open(filename, std::ios_base::app); + if (!telemetry_file_.is_open()) { + LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; + } +} + +void FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_shard_id(broadcast->dst_shard), create_tl_block_id(broadcast->block_id), + create_tl_object(broadcast->max_bytes, broadcast->max_msgs), + create_tl_object(broadcast->queue_proofs.clone(), + broadcast->block_state_proofs.clone(), + std::vector(1, broadcast->msg_count))); + VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay to " << broadcast->dst_shard.to_str() + << " from " << broadcast->block_id.to_str() << ", msgs=" << broadcast->msg_count + << " bytes=" << broadcast->queue_proofs.size(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + +void FullNodeFastSyncOverlay::start_up() { + auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodeFastSyncOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodeFastSyncOverlay::init() { + LOG(INFO) << "Creating fast sync overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::get_stats_extra, std::move(promise)); + } + explicit Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, + {}}; + std::string scope = PSTRING() << R"({ "type": "fast-sync", "shard_id": )" << shard_.shard + << ", \"workchain_id\": " << shard_.workchain << " }"; + overlay::OverlayOptions options; + bool is_validator = std::find(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id_) != + current_validators_adnl_.end(); + if (!shard_.is_masterchain()) { + options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + } + options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + options.max_slaves_in_semiprivate_overlay_ = FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS; + td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, + overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, + std::make_unique(actor_id(this)), rules, std::move(scope), options); + + inited_ = true; + if (shard_.is_masterchain()) { + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor( + "telemetry", local_id_, std::make_unique(actor_id(this))); + } +} + +void FullNodeFastSyncOverlay::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodeFastSyncOverlay::set_validators(std::vector root_public_keys, + std::vector current_validators_adnl) { + root_public_keys_ = std::move(root_public_keys); + current_validators_adnl_ = std::move(current_validators_adnl); + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + +void FullNodeFastSyncOverlay::set_member_certificate(overlay::OverlayMemberCertificate member_certificate) { + member_certificate_ = std::move(member_certificate); + if (inited_) { + td::actor::send_closure(overlays_, &overlay::Overlays::update_member_certificate, local_id_, overlay_id_, + member_certificate_); + } +} + +void FullNodeFastSyncOverlay::set_receive_broadcasts(bool value) { + if (value == receive_broadcasts_) { + return; + } + receive_broadcasts_ = value; + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + +void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : current_validators_adnl_) { + res->validators_adnl_.push_back(x.bits256_value()); + } + for (const auto &x : root_public_keys_) { + res->root_public_keys_.push_back(x.bits256_value()); + } + if (!member_certificate_.empty()) { + res->member_certificate_ = member_certificate_.tl(); + } + res->receive_broadcasts_ = receive_broadcasts_; + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +std::pair, adnl::AdnlNodeIdShort> FullNodeFastSyncOverlays::choose_overlay( + ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end()) { + return {it->second.get(), p.first}; + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {td::actor::ActorId{}, adnl::AdnlNodeIdShort::zero()}; +} + +td::actor::ActorId FullNodeFastSyncOverlays::get_masterchain_overlay_for( + adnl::AdnlNodeIdShort adnl_id) { + auto it = id_to_overlays_.find(adnl_id); + if (it == id_to_overlays_.end()) { + return {}; + } + auto it2 = it->second.overlays_.find(ShardIdFull{masterchainId}); + if (it2 == it->second.overlays_.end()) { + return {}; + } + return it2->second.get(); +} + +void FullNodeFastSyncOverlays::update_overlays(td::Ref state, + std::set my_adnl_ids, + std::set monitoring_shards, + const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, + const td::actor::ActorId &adnl, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager, + const td::actor::ActorId &full_node) { + monitoring_shards.insert(ShardIdFull{masterchainId}); + std::set all_shards; + all_shards.insert(ShardIdFull{masterchainId}); + for (const auto &desc : state->get_shards()) { + ShardIdFull shard = desc->shard(); + td::uint32 monitor_min_split = state->monitor_min_split_depth(shard.workchain); + if (shard.pfx_len() > monitor_min_split) { + shard = shard_prefix(shard, monitor_min_split); + } + all_shards.insert(shard); + } + + // Remove overlays for removed adnl ids and shards + for (auto it = id_to_overlays_.begin(); it != id_to_overlays_.end();) { + if (my_adnl_ids.count(it->first)) { + auto &overlays_info = it->second; + for (auto it2 = overlays_info.overlays_.begin(); it2 != overlays_info.overlays_.end();) { + if (all_shards.count(it2->first)) { + ++it2; + } else { + it2 = overlays_info.overlays_.erase(it2); + } + } + ++it; + } else { + it = id_to_overlays_.erase(it); + } + } + + // On new keyblock - update validator set + bool updated_validators = false; + if (!last_key_block_seqno_ || last_key_block_seqno_.value() != state->last_key_block_id().seqno()) { + updated_validators = true; + last_key_block_seqno_ = state->last_key_block_id().seqno(); + root_public_keys_.clear(); + current_validators_adnl_.clear(); + // Previous, current and next validator sets + for (int i = -1; i <= 1; ++i) { + auto val_set = state->get_total_validator_set(i); + if (val_set.is_null()) { + continue; + } + for (const ValidatorDescr &val : val_set->export_vector()) { + PublicKeyHash public_key_hash = ValidatorFullId{val.key}.compute_short_id(); + root_public_keys_.push_back(public_key_hash); + current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); + } + } + std::sort(root_public_keys_.begin(), root_public_keys_.end()); + root_public_keys_.erase(std::unique(root_public_keys_.begin(), root_public_keys_.end()), root_public_keys_.end()); + std::sort(current_validators_adnl_.begin(), current_validators_adnl_.end()); + current_validators_adnl_.erase(std::unique(current_validators_adnl_.begin(), current_validators_adnl_.end()), + current_validators_adnl_.end()); + + for (auto &[local_id, overlays_info] : id_to_overlays_) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + for (auto &[shard, overlay] : overlays_info.overlays_) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_validators, root_public_keys_, + current_validators_adnl_); + } + } + } + + // Cleanup outdated certificates + double now = td::Clocks::system(); + for (auto &[_, certificates] : member_certificates_) { + certificates.erase(std::remove_if(certificates.begin(), certificates.end(), + [&](const overlay::OverlayMemberCertificate &certificate) { + return certificate.is_expired(now); + }), + certificates.end()); + } + + for (adnl::AdnlNodeIdShort local_id : my_adnl_ids) { + bool is_new = !id_to_overlays_.count(local_id); + auto &overlays_info = id_to_overlays_[local_id]; + // Update is_validator and current_certificate + if (is_new) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + } + bool changed_certificate = false; + // Check if certificate is outdated or no longer authorized by current root keys + if (!overlays_info.current_certificate_.empty() && overlays_info.current_certificate_.is_expired(now)) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (!overlays_info.current_certificate_.empty() && updated_validators && + !std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + overlays_info.current_certificate_.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (overlays_info.current_certificate_.empty()) { + auto it = member_certificates_.find(local_id); + if (it != member_certificates_.end()) { + for (const overlay::OverlayMemberCertificate &certificate : it->second) { + if (std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + certificate.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = it->second.front(); + break; + } + } + } + } + + // Remove if it is not authorized + if (!overlays_info.is_validator_ && overlays_info.current_certificate_.empty()) { + id_to_overlays_.erase(local_id); + continue; + } + + // Update shard overlays + for (ShardIdFull shard : all_shards) { + bool receive_broadcasts = monitoring_shards.contains(shard); + auto &overlay = overlays_info.overlays_[shard]; + if (overlay.empty()) { + overlay = td::actor::create_actor( + PSTRING() << "FastSyncOv" << shard.to_str(), local_id, shard, zero_state_file_hash, root_public_keys_, + current_validators_adnl_, overlays_info.current_certificate_, receive_broadcasts, keyring, adnl, overlays, + validator_manager, full_node); + } else { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_receive_broadcasts, receive_broadcasts); + if (changed_certificate) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, + overlays_info.current_certificate_); + } + } + } + } +} + +void FullNodeFastSyncOverlays::add_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate member_certificate) { + if (member_certificate.empty() || member_certificate.is_expired()) { + return; + } + member_certificates_[local_id].push_back(std::move(member_certificate)); + // Overlays will be updated in the next update_overlays +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp new file mode 100644 index 000000000..3505ae5eb --- /dev/null +++ b/validator/full-node-fast-sync-overlays.hpp @@ -0,0 +1,145 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node.h" +#include "validator-telemetry.hpp" + +#include + +namespace ton::validator::fullnode { + +class FullNodeFastSyncOverlay : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast& query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); + + void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, + const tl_object_ptr& telemetry); + + template + void process_broadcast(PublicKeyHash, T&) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); + void send_out_msg_queue_proof_broadcast(td::Ref broadcast); + void send_validator_telemetry(tl_object_ptr telemetry); + + void collect_validator_telemetry(std::string filename); + + void start_up() override; + void tear_down() override; + + void set_validators(std::vector root_public_keys, + std::vector current_validators_adnl); + void set_member_certificate(overlay::OverlayMemberCertificate member_certificate); + void set_receive_broadcasts(bool value); + + FullNodeFastSyncOverlay(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, FileHash zero_state_file_hash, + std::vector root_public_keys, + std::vector current_validators_adnl, + overlay::OverlayMemberCertificate member_certificate, bool receive_broadcasts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) + : local_id_(local_id) + , shard_(shard) + , root_public_keys_(std::move(root_public_keys)) + , current_validators_adnl_(std::move(current_validators_adnl)) + , member_certificate_(std::move(member_certificate)) + , receive_broadcasts_(receive_broadcasts) + , zero_state_file_hash_(zero_state_file_hash) + , keyring_(keyring) + , adnl_(adnl) + , overlays_(overlays) + , validator_manager_(validator_manager) + , full_node_(full_node) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + ShardIdFull shard_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; + overlay::OverlayMemberCertificate member_certificate_; + bool receive_broadcasts_; + FileHash zero_state_file_hash_; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + UnixTime created_at_ = (UnixTime)td::Clocks::system(); + + void try_init(); + void init(); + void get_stats_extra(td::Promise promise); + + td::actor::ActorOwn telemetry_sender_; + bool collect_telemetry_ = false; + std::ofstream telemetry_file_; +}; + +class FullNodeFastSyncOverlays { + public: + std::pair, adnl::AdnlNodeIdShort> choose_overlay(ShardIdFull shard); + td::actor::ActorId get_masterchain_overlay_for(adnl::AdnlNodeIdShort adnl_id); + void update_overlays(td::Ref state, std::set my_adnl_ids, + std::set monitoring_shards, const FileHash& zero_state_file_hash, + const td::actor::ActorId& keyring, const td::actor::ActorId& adnl, + const td::actor::ActorId& overlays, + const td::actor::ActorId& validator_manager, + const td::actor::ActorId& full_node); + void add_member_certificate(adnl::AdnlNodeIdShort local_id, overlay::OverlayMemberCertificate member_certificate); + + private: + struct Overlays { + std::map> overlays_; + overlay::OverlayMemberCertificate current_certificate_; + bool is_validator_{false}; + }; + + std::map id_to_overlays_; // local_id -> overlays + std::map> member_certificates_; + + td::optional last_key_block_seqno_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; +}; + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index da49f0e2e..7037f8396 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -275,21 +275,26 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, - masterchain_block_id, std::move(P)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -349,24 +354,14 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, @@ -389,6 +384,21 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo ShardIdFull{masterchainId}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, @@ -405,6 +415,38 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object()); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, + ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + mc_block_id, state_type, std::move(P)); +} + void FullNodeMasterImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { auto BX = fetch_tl_prefix(query, true); diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index ce0aedd35..82815b48f 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,6 +66,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -80,8 +82,16 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); + + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); + // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index f86323fc0..7ea813a46 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -34,6 +34,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -49,8 +54,8 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, @@ -63,6 +68,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; @@ -89,7 +99,7 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas } void FullNodePrivateBlockOverlay::process_telemetry_broadcast( - PublicKeyHash src, const tl_object_ptr& telemetry) { + PublicKeyHash src, const tl_object_ptr &telemetry) { if (telemetry->adnl_id_ != src.bits256_value()) { VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; return; @@ -107,9 +117,7 @@ void FullNodePrivateBlockOverlay::process_telemetry_broadcast( } VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; auto s = td::json_encode(td::ToJson(*telemetry), false); - std::erase_if(s, [](char c) { - return c == '\n' || c == '\r'; - }); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); telemetry_file_ << s << "\n"; telemetry_file_.flush(); if (telemetry_file_.fail()) { @@ -131,9 +139,7 @@ void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::Buffe } return; } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { - Self->process_broadcast(src, obj); - }); + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); } void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -185,7 +191,9 @@ void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { } void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + } auto data = serialize_tl_object(telemetry, true); if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, @@ -266,6 +274,7 @@ void FullNodePrivateBlockOverlay::init() { {}}; overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", overlay_options); @@ -273,6 +282,20 @@ void FullNodePrivateBlockOverlay::init() { td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); inited_ = true; + + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor("telemetry", local_id_, + std::make_unique(actor_id(this))); } void FullNodePrivateBlockOverlay::tear_down() { @@ -289,6 +312,10 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod process_block_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " @@ -327,6 +354,11 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 70e196ea6..2cbe25f68 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -25,12 +27,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); @@ -98,6 +102,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void try_init(); void init(); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; @@ -106,12 +111,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); template diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 94dc2155e..5665cc233 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -20,6 +20,7 @@ #include "auto/tl/ton_api.hpp" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" +#include "vm/boc-compression.h" #include "td/utils/lz4.h" #include "full-node.h" #include "td/utils/overloaded.h" @@ -88,6 +89,28 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b std::move(proof)}; } +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressedV2& f, + int max_decompressed_size) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size) { td::Result B; @@ -96,6 +119,9 @@ td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcas [&](ton_api::tonNode_blockBroadcastCompressed& f) { B = deserialize_block_broadcast(f, max_decompressed_data_size); }, + [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); return B; } @@ -139,6 +165,20 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, return td::Status::OK(); } +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { td::Status S; @@ -148,6 +188,9 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id [&](ton_api::tonNode_dataFullCompressed& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); }, + [&](ton_api::tonNode_dataFullCompressedV2& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } @@ -194,6 +237,24 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc return td::Status::OK(); } +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(roots, vm::boc_decompress(obj.compressed_, max_decompressed_data_size)); + if (roots.size() != 1) { + return td::Status::Error("expected 1 root in boc"); + } + auto root = std::move(roots[0]); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast V2: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, int max_decompressed_data_size) { @@ -207,6 +268,10 @@ td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, data, max_decompressed_data_size); }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ac0eb7688..771b28be3 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -509,13 +509,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); @@ -523,8 +522,14 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto masterchain_block_id = create_block_id(query.masterchain_block_); VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, - masterchain_block_id, std::move(P)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -536,7 +541,11 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto P = td::PromiseCreator::lambda([promise = std::move(promise), cnt](td::Result> R) mutable { if (R.is_error()) { - LOG(WARNING) << "getnextkey: " << R.move_as_error(); + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "getnextkey: " << R.move_as_error(); + } else { + LOG(WARNING) << "getnextkey: " << R.move_as_error(); + } auto x = create_serialize_tl_object( std::vector>{}, false, true); promise.set_value(std::move(x)); @@ -593,31 +602,14 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, 0, max_size + 1, std::move(P)); + masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, td::Promise promise) { - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " - << src; - if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); - return; - } - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, @@ -692,14 +684,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod return; } block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; - if (limits.max_msgs > 512) { - promise.set_error(td::Status::Error("max_msgs is too big")); - return; - } - if (limits.max_bytes > (1 << 21)) { - promise.set_error(td::Status::Error("max_bytes is too big")); - return; - } FLOG(DEBUG) { sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; for (const BlockIdExt &id : blocks) { @@ -727,6 +711,47 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise) { + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") " + << query.offset_ << " " << query.max_size_ << " from " << src; + if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") from " + << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + mc_block_id, state_type, std::move(P)); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { @@ -766,8 +791,8 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { @@ -779,6 +804,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; @@ -807,6 +837,10 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl process_block_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -951,17 +985,19 @@ void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::T void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, adnl_id_, - overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, + UnsplitStateType{}, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) .release(); } -void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { +void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, + type, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -978,9 +1014,9 @@ void FullNodeShardImpl::download_block_proof(BlockIdExt block_id, td::uint32 pri void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - b.adnl_id, priority, timeout, validator_manager_, rldp_, - overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, b.adnl_id, + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } @@ -1006,7 +1042,7 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) - auto &b = choose_neighbour(); + auto &b = choose_neighbour(3, 0); // Required version: 3.0 if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; @@ -1253,24 +1289,36 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour() const { +const Neighbour &FullNodeShardImpl::choose_neighbour(td::uint32 required_version_major, + td::uint32 required_version_minor) const { if (neighbours_.size() == 0) { return Neighbour::zero; } + auto is_eligible = + [&](const Neighbour &n) { + return n.version_major > required_version_major || + (n.version_major == required_version_major && n.version_minor >= required_version_minor); + }; double min_unreliability = 1e9; - for (auto &x : neighbours_) { - min_unreliability = std::min(min_unreliability, x.second.unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + min_unreliability = std::min(min_unreliability, x.unreliability); } const Neighbour *best = nullptr; td::uint32 sum = 0; - for (auto &x : neighbours_) { - auto unr = static_cast(x.second.unreliability - min_unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + auto unr = static_cast(x.unreliability - min_unreliability); - if (x.second.version_major < proto_version_major()) { + if (x.version_major < proto_version_major()) { unr += 4; - } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { + } else if (x.version_major == proto_version_major() && x.version_minor < proto_version_minor()) { unr += 2; } @@ -1280,7 +1328,7 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { auto w = 1 << (f - unr); sum += w; if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; + best = &x; } } } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 5898db80d..0c6aa441b 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -55,8 +55,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fb3eef769..a90460adb 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -123,6 +123,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -141,19 +143,28 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + LOG(ERROR) << "Ignore outMsgQueueProofBroadcast"; + } void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); @@ -172,8 +183,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override; + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override; void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; @@ -206,10 +218,10 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour() const; + const Neighbour &choose_neighbour(td::uint32 required_version_major = 0, td::uint32 required_version_minor = 0) const; template - td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p) { + td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now()](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e1951c365..d648fc821 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -96,6 +96,16 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr promise.set_value(td::Unit()); } +void FullNodeImpl::add_collator_adnl_id(adnl::AdnlNodeIdShort id) { + ++local_collator_nodes_[id]; +} + +void FullNodeImpl::del_collator_adnl_id(adnl::AdnlNodeIdShort id) { + if (--local_collator_nodes_[id] == 0) { + local_collator_nodes_.erase(id); + } +} + void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { auto it = shards_.find(shard_id); @@ -251,6 +261,32 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std shard_info.delete_at = td::Timestamp::never(); } } + + int proto_version = state->get_consensus_config().proto_version; + use_old_private_overlays_ = (proto_version < 5); + + if (!use_old_private_overlays_) { + private_block_overlays_.clear(); + std::set my_adnl_ids; + my_adnl_ids.insert(adnl_id_); + for (const auto &[adnl_id, _] : local_collator_nodes_) { + my_adnl_ids.insert(adnl_id); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } + } + std::set monitoring_shards; + for (ShardIdFull shard : shards_to_monitor) { + monitoring_shards.insert(cut_shard(shard)); + } + fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), + zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, + actor_id(this)); + update_validator_telemetry_collector(); + } } void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { @@ -282,21 +318,28 @@ void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(dst); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; - return; - } + bool skip_public = false; for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { for (auto &[local_id, actor] : private_overlay.actors_) { if (private_overlay.params_.msg_senders_.contains(local_id)) { td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + if (private_overlay.params_.skip_public_msg_send_) { + skip_public = true; + } } } } } - td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + + if (!skip_public) { + auto shard = get_shard(dst); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; + return; + } + td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + } } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { @@ -309,43 +352,69 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, + data.clone()); + } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; - return; + td::BufferSlice data, int mode) { + if (mode & broadcast_mode_custom) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, - block_id, cc_seqno, validator_set_hash, data.clone()); + if (mode & broadcast_mode_private_block) { + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } } - if (broadcast_block_candidates_in_public_overlay_) { + if (mode & broadcast_mode_public) { + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, std::move(data)); } } +void FullNodeImpl::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast->dst_shard).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast, + std::move(broadcast)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); } - auto shard = get_shard(broadcast.block_id.shard_full()); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; - return; - } if (mode & broadcast_mode_private_block) { if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); + } } if (mode & broadcast_mode_public) { + auto shard = get_shard(broadcast.block_id.shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } } @@ -372,16 +441,17 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T td::actor::send_closure(shard, &FullNodeShard::download_zero_state, id, priority, timeout, std::move(promise)); } -void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(id.shard_full()); +void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { + auto shard = get_shard(id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, priority, timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, type, priority, + timeout, std::move(promise)); } void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -397,7 +467,7 @@ void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority void FullNodeImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(block_id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download proof link query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -420,7 +490,7 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(shard_prefix); + auto shard = get_shard(shard_prefix, /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -449,7 +519,7 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect timeout, std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool historical) { if (shard.is_masterchain()) { return shards_[ShardIdFull{masterchainId}].actor.get(); } @@ -457,13 +527,23 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { return {}; } int pfx_len = shard.pfx_len(); - if (pfx_len > wc_monitor_min_split_) { - shard = shard_prefix(shard, wc_monitor_min_split_); + int min_split = wc_monitor_min_split_; + if (historical) { + min_split = td::Random::fast(0, min_split); + } + if (pfx_len > min_split) { + shard = shard_prefix(shard, min_split); } - auto it = shards_.find(shard); - if (it != shards_.end()) { - update_shard_actor(shard, it->second.active); - return it->second.actor.get(); + while (true) { + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); } // Special case if shards_ was not yet initialized. @@ -545,18 +625,9 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } -void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; - } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); -} - void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { send_block_broadcast_to_custom_overlays(broadcast); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_broadcast, std::move(broadcast), [](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -572,7 +643,7 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha td::uint32 validator_set_hash, td::BufferSlice data) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); // ignore cc_seqno and validator_hash for now - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, std::move(data)); } @@ -587,15 +658,31 @@ void FullNodeImpl::set_validator_telemetry_filename(std::string value) { } void FullNodeImpl::update_validator_telemetry_collector() { - if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { - validator_telemetry_collector_key_ = PublicKeyHash::zero(); - return; - } - if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { - auto it = private_block_overlays_.begin(); - validator_telemetry_collector_key_ = it->first; - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, - validator_telemetry_filename_); + if (use_old_private_overlays_) { + if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { + auto it = private_block_overlays_.begin(); + validator_telemetry_collector_key_ = it->first; + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } else { + if (validator_telemetry_filename_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{validator_telemetry_collector_key_}) + .empty()) { + auto [actor, adnl_id] = fast_sync_overlays_.choose_overlay(ShardIdFull{masterchainId}); + validator_telemetry_collector_key_ = adnl_id.pubkey_hash(); + if (!actor.empty()) { + td::actor::send_closure(actor, &FullNodeFastSyncOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } } } @@ -630,9 +717,12 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, - std::move(data)); + std::move(data), mode); + } + void send_out_msg_queue_proof_broadcast(td::Ref broadcast) override { + td::actor::send_closure(id_, &FullNodeImpl::send_out_msg_queue_proof_broadcast, std::move(broadcast)); } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); @@ -645,9 +735,10 @@ void FullNodeImpl::start_up() { td::Promise promise) override { td::actor::send_closure(id_, &FullNodeImpl::download_zero_state, id, priority, timeout, std::move(promise)); } - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, priority, + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, type, priority, timeout, std::move(promise)); } void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -679,9 +770,6 @@ void FullNodeImpl::start_up() { void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); - } explicit Callback(td::actor::ActorId id) : id_(id) { } @@ -710,6 +798,9 @@ void FullNodeImpl::update_private_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { + if (!use_old_private_overlays_) { + return; + } CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; @@ -863,6 +954,7 @@ CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_c for (const auto &shard : f.sender_shards_) { c.sender_shards_.push_back(create_shard_id(shard)); } + c.skip_public_msg_send_ = f.skip_public_msg_send_; return c; } diff --git a/validator/full-node.h b/validator/full-node.h index 555082dc5..4c11c1fbd 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -67,6 +67,7 @@ struct CustomOverlayParams { std::map msg_senders_; std::set block_senders_; std::vector sender_shards_; + bool skip_public_msg_send_ = false; bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); @@ -80,6 +81,8 @@ class FullNode : public td::actor::Actor { virtual void add_permanent_key(PublicKeyHash key, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; + virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, @@ -101,6 +104,9 @@ class FullNode : public td::actor::Actor { virtual void set_validator_telemetry_filename(std::string value) = 0; + virtual void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) = 0; + static constexpr td::uint32 max_block_size() { return 4 << 20; } @@ -112,6 +118,8 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5; + static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index b4c79363e..482a916bb 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,6 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" +#include "full-node-fast-sync-overlays.hpp" #include #include @@ -44,6 +45,8 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; + void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; @@ -66,13 +69,14 @@ class FullNodeImpl : public FullNode { void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data); + td::BufferSlice data, int mode); void send_broadcast(BlockBroadcast broadcast, int mode); + void send_out_msg_queue_proof_broadcast(td::Ref broadcats); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise); + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -86,7 +90,6 @@ class FullNodeImpl : public FullNode { void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, @@ -95,6 +98,14 @@ class FullNodeImpl : public FullNode { void set_validator_telemetry_filename(std::string value) override; + void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) override { + VLOG(FULL_NODE_DEBUG) << "Importing fast sync overlay certificate for " << local_id << " issued by " + << cert.issued_by().compute_short_id() << " expires in " + << (double)cert.expire_at() - td::Clocks::system(); + fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); + } + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -119,7 +130,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard); + td::actor::ActorId get_shard(ShardIdFull shard, bool historical = false); std::map shards_; int wc_monitor_min_split_ = 0; @@ -139,12 +150,18 @@ class FullNodeImpl : public FullNode { std::map current_validators_; std::set local_keys_; + std::map local_collator_nodes_; td::Promise started_promise_; FullNodeOptions opts_; + // Private overlays: + // Old overlays - one private overlay for all validators + // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) + // for validators and authorized nodes + bool use_old_private_overlays_ = true; std::map> private_block_overlays_; - bool broadcast_block_candidates_in_public_overlay_ = false; + FullNodeFastSyncOverlays fast_sync_overlays_; struct CustomOverlayInfo { CustomOverlayParams params_; diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 978cf859a..9ab32c47b 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index de48626d7..883994089 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -41,7 +41,7 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, + td::Ref approve_signatures, int send_broadcast_mode, bool apply, td::actor::ActorId manager, td::Promise promise) : id_(id) , data_(std::move(data)) @@ -52,6 +52,7 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , is_fake_(false) , is_fork_(false) , send_broadcast_mode_(send_broadcast_mode) + , apply_(apply) , manager_(manager) , promise_(std::move(promise)) , perf_timer_("acceptblock", 0.1, [manager](double duration) { @@ -150,6 +151,7 @@ bool AcceptBlockQuery::precheck_header() { if (is_fork_ && !info.key_block) { return fatal_error("fork block is not a key block"); } + before_split_ = info.before_split; return true; } @@ -348,7 +350,9 @@ bool AcceptBlockQuery::check_send_error(td::actor::ActorId Sel } void AcceptBlockQuery::finish_query() { - ValidatorInvariants::check_post_accept(handle_); + if (apply_) { + ValidatorInvariants::check_post_accept(handle_); + } if (is_masterchain()) { CHECK(handle_->inited_proof()); } else { @@ -490,6 +494,21 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + block_root_ = data_->root_cell(); + if (block_root_.is_null()) { + fatal_error("block data does not contain a root cell"); + return; + } + // generate proof + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + send_broadcasts(); + if (!apply_) { + written_state({}); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_prev_state, R.move_as_ok()); @@ -555,24 +574,14 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { CHECK(data_.not_null()); state_ = std::move(upd_state); - block_root_ = data_->root_cell(); - if (block_root_.is_null()) { - fatal_error("block data does not contain a root cell"); - return; - } - // generate proof - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - - if (state_keep_old_hash_ != state_old_hash_) { + if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); return; } //handle_->set_masterchain_block(prev_[0]); + handle_->set_split(before_split_); handle_->set_state_root_hash(state_hash_); handle_->set_logical_time(lt_); handle_->set_unix_time(created_at_); @@ -617,7 +626,7 @@ void AcceptBlockQuery::got_last_mc_block(std::pair, Bl VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str() << ", trying to obtain it"; td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_, - [SelfId = actor_id(this)](td::Result> R) { + false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state, R.move_as_ok()); @@ -926,8 +935,11 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { + finish_query(); +} + +void AcceptBlockQuery::send_broadcasts() { if (send_broadcast_mode_ == 0) { - finish_query(); return; } BlockBroadcast b; @@ -951,7 +963,10 @@ void AcceptBlockQuery::applied() { // do not wait for answer td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), send_broadcast_mode_); - finish_query(); + // Do this for shard blocks later: + // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, + // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + // std::move(b.data), send_broadcast_mode_); } } // namespace validator diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index d1c0baa6f..e582650c1 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -50,7 +50,7 @@ class AcceptBlockQuery : public td::actor::Actor { struct ForceFork {}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, + td::Ref approve_signatures, int send_broadcast_mode, bool apply, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, @@ -89,6 +89,7 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_next(); void written_block_info_2(); void applied(); + void send_broadcasts(); private: BlockIdExt id_; @@ -100,6 +101,7 @@ class AcceptBlockQuery : public td::actor::Actor { bool is_fake_; bool is_fork_; int send_broadcast_mode_{0}; + bool apply_ = true; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; @@ -116,6 +118,7 @@ class AcceptBlockQuery : public td::actor::Actor { UnixTime created_at_; RootHash state_keep_old_hash_, state_old_hash_, state_hash_; BlockIdExt mc_blkid_, prev_mc_blkid_; + bool before_split_; Ref last_mc_state_; BlockIdExt last_mc_id_; diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 30a13c089..12d336ad1 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -344,7 +344,7 @@ void CheckProof::got_block_handle(BlockHandle handle) { process_masterchain_state(); return; } - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_masterchain_state, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 340e3a401..83ae4370a 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "shard.hpp" #include "top-shard-descr.hpp" @@ -33,6 +34,7 @@ #include #include #include "common/global-version.h" +#include "fabric.h" namespace ton { @@ -40,13 +42,16 @@ namespace validator { using td::Ref; class Collator final : public td::actor::Actor { + public: static constexpr int supported_version() { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } + + private: using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; @@ -76,7 +81,9 @@ class Collator final : public td::actor::Actor { td::Timestamp timeout; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; - unsigned mode_ = 0; + adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); + bool skip_store_candidate_ = false; + Ref optimistic_prev_block_; int attempt_idx_; bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; @@ -91,10 +98,8 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, - Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); + Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; @@ -113,13 +118,16 @@ class Collator final : public td::actor::Actor { Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, - LogicalTime after_lt); + LogicalTime after_lt, CollationStats* stats = nullptr); private: void start_up() override; + void load_prev_states_blocks(); + bool process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; + bool full_collated_data_ = false; ton::LogicalTime start_lt, max_lt; ton::UnixTime now_; ton::UnixTime prev_now_; @@ -134,6 +142,7 @@ class Collator final : public td::actor::Actor { std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; + std::map neighbor_msg_queues_limits_; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; @@ -149,6 +158,7 @@ class Collator final : public td::actor::Actor { ton::LogicalTime shards_max_end_lt_{0}; ton::UnixTime prev_state_utime_; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno min_ref_mc_seqno_{~0U}; ton::BlockSeqno vert_seqno_{~0U}, prev_vert_seqno_{~0U}; ton::BlockIdExt prev_key_block_; @@ -178,6 +188,7 @@ class Collator final : public td::actor::Actor { td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + vm::ProofStorageStat collated_data_stat; int block_limit_class_ = 0; ton::LogicalTime min_new_msg_lt{std::numeric_limits::max()}; block::CurrencyCollection total_balance_, old_total_balance_, total_validator_fees_; @@ -195,7 +206,10 @@ class Collator final : public td::actor::Actor { std::vector ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; - std::unique_ptr in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_; + block::tlb::Aug_InMsgDescr aug_InMsgDescr{0}; + block::tlb::Aug_OutMsgDescr aug_OutMsgDescr{0}; + std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, + sibling_out_msg_queue_; std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs td::uint64 out_msg_queue_size_ = 0; td::uint64 old_out_msg_queue_size_ = 0; @@ -207,10 +221,23 @@ class Collator final : public td::actor::Actor { unsigned block_create_total_{0}; std::vector bad_ext_msgs_, delay_ext_msgs_; Ref shard_account_blocks_; // ShardAccountBlocks + + std::map> block_state_proofs_; + std::vector neighbor_proof_builders_; std::vector> collated_roots_; + + struct AccountStorageDict { + bool inited = false; + vm::MerkleProofBuilder mpb; + vm::ProofStorageStat proof_stat; + bool add_to_collated_data = false; + std::vector> storage_stat_updates; + }; + std::map account_storage_dicts_; + std::unique_ptr block_candidate; - std::unique_ptr dispatch_queue_; + std::unique_ptr dispatch_queue_, old_dispatch_queue_; std::map sender_generated_messages_count_; unsigned dispatch_queue_ops_{0}; std::map last_dispatch_queue_emitted_lt_; @@ -227,11 +254,16 @@ class Collator final : public td::actor::Actor { bool deferring_messages_enabled_ = false; bool store_out_msg_queue_size_ = false; + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; + td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, bool force_create); + bool init_account_storage_dict(block::Account& account); td::Result make_account(td::ConstBitPtr addr, bool force_create = false); td::actor::ActorId get_self() { return actor_id(this); @@ -243,18 +275,22 @@ class Collator final : public td::actor::Actor { bool fatal_error(int err_code, std::string err_msg); bool fatal_error(std::string err_msg, int err_code = -666); void check_pending(); - void after_get_mc_state(td::Result, BlockIdExt>> res); - void after_get_shard_state(int idx, td::Result> res); - void after_get_block_data(int idx, td::Result> res); - void after_get_shard_blocks(td::Result>> res); + void after_get_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); + void after_get_block_data(int idx, td::Result> res, td::PerfLogAction token); + void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token); + void got_neighbor_msg_queue(unsigned i, Ref res); void got_out_queue_size(size_t i, td::Result res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, @@ -264,6 +300,7 @@ class Collator final : public td::actor::Actor { bool register_shard_block_creators(std::vector creator_list); bool init_block_limits(); bool compute_minted_amount(block::CurrencyCollection& to_mint); + bool create_output_queue_merger(); bool init_value_create(); bool try_collate(); bool do_preinit(); @@ -295,10 +332,18 @@ class Collator final : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; - void after_get_external_messages(td::Result, int>>> res); + void after_get_external_messages(td::Result, int>>> res, td::PerfLogAction token); td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, int priority); // td::Result register_external_message(td::Slice ext_msg_boc); @@ -307,8 +352,8 @@ class Collator final : public td::actor::Actor { bool process_new_messages(bool enqueue_only = false); int process_one_new_message(block::NewOutMsg msg, bool enqueue_only = false, Ref* is_special = nullptr); bool process_inbound_internal_messages(); - bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb); + bool precheck_inbound_message(Ref msg, ton::LogicalTime lt); + bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, int src_nb_idx); bool process_inbound_external_messages(); int process_external_message(Ref msg); bool process_dispatch_queue(); @@ -327,7 +372,9 @@ class Collator final : public td::actor::Actor { bool register_out_msg_queue_op(bool force = false); bool register_dispatch_queue_op(bool force = false); bool update_account_dict_estimation(const block::transaction::Transaction& trans); + void update_account_storage_dict_info(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); + bool process_account_storage_dict(block::Account& account); bool combine_account_transactions(); bool update_public_libraries(); bool update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr); @@ -355,11 +402,13 @@ class Collator final : public td::actor::Actor { bool update_cc); bool create_mc_block_extra(Ref& mc_block_extra); bool create_block(); + Ref collate_shard_block_descr_set(); + bool prepare_msg_queue_proof(); bool create_collated_data(); bool create_block_candidate(); - void return_block_candidate(td::Result saved); + void return_block_candidate(td::Result saved, td::PerfLogAction token); bool update_last_proc_int_msg(const std::pair& new_lt_hash); td::CancellationToken cancellation_token_; @@ -369,9 +418,15 @@ class Collator final : public td::actor::Actor { static td::uint32 get_skip_externals_queue_size(); private: - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; CollationStats stats_; + + void finalize_stats(); + + AccountStorageDict* current_tx_storage_dict_ = nullptr; + + void on_cell_loaded(const vm::LoadedCell& cell); + void set_current_tx_storage_dict(const block::Account& account); }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2a6d7a2b2..14c2d9aed 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "candidate-serializer.h" #include "collator-impl.h" #include "vm/boc.h" #include "td/db/utils/BlobView.h" @@ -31,6 +32,7 @@ #include #include #include "fabric.h" +#include "storage-stat-cache.hpp" #include "validator-set.hpp" #include "top-shard-descr.hpp" #include @@ -56,32 +58,21 @@ static constexpr int MAX_ATTEMPTS = 5; /** * Constructs a Collator object. * - * @param shard The shard of the new block. - * @param is_hardfork A boolean indicating whether the new block is a hardfork. - * @param min_masterchain_block_id The the minimum reference masterchain block. - * @param prev A vector of BlockIdExt representing the previous blocks. - * @param validator_set A reference to the ValidatorSet. - * @param collator_id The public key of the block creator. - * @param collator_opts A reference to CollatorOptions. + * @param params Collator parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. - * @param promise The promise to return the result. * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk. - * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. - */ -Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, - unsigned mode, int attempt_idx) - : shard_(shard) - , is_hardfork_(is_hardfork) - , min_mc_block_id{min_masterchain_block_id} - , prev_blocks(std::move(prev)) - , created_by_(collator_id) - , collator_opts_(collator_opts) - , validator_set_(std::move(validator_set)) + * @param promise The promise to return the result. + */ +Collator::Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) + : shard_(params.shard) + , is_hardfork_(params.is_hardfork) + , min_mc_block_id{params.min_masterchain_block_id} + , prev_blocks(std::move(params.prev)) + , created_by_(params.creator) + , collator_opts_(params.collator_opts.is_null() ? td::Ref{true} : params.collator_opts) + , validator_set_(std::move(params.validator_set)) , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query @@ -89,8 +80,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) - , mode_(mode) - , attempt_idx_(attempt_idx) + , collator_node_id_(params.collator_node_id) + , skip_store_candidate_(params.skip_store_candidate) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) + , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); @@ -200,57 +193,56 @@ void Collator::start_up() { return; } } + if (optimistic_prev_block_.not_null()) { + if (prev_blocks.size() != 1) { + fatal_error(-666, "optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error(-666, "optimistic prev block is not null, but has invalid block id"); + return; + } + } busy_ = true; step = 1; if (!is_masterchain()) { // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; + auto token = perf_log_.start_action("get_top_masterchain_state_block"); if (!is_hardfork_) { td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = std::move(token)]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::move(res)); + std::move(res), std::move(token)); }); } else { - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, - [self = get_self(), block_id = min_mc_block_id](td::Result> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - if (res.is_error()) { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, res.move_as_error()); - } else { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::make_pair(Ref(res.move_as_ok()), block_id)); - } - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, + [self = get_self(), block_id = min_mc_block_id, + token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + if (res.is_error()) { + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, + res.move_as_error(), std::move(token)); + } else { + td::actor::send_closure_later( + std::move(self), &Collator::after_get_mc_state, + std::make_pair(Ref(res.move_as_ok()), block_id), + std::move(token)); + } + }); } } // 3. load previous block(s) and corresponding state(s) prev_states.resize(prev_blocks.size()); prev_block_data.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, - std::move(res)); - }); - if (prev_blocks[i].seqno()) { - // 3.2. load block - // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs - LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, - std::move(res)); - }); + if (optimistic_prev_block_.is_null()) { + load_prev_states_blocks(); + } else { + if (!process_optimistic_prev_block()) { + return; } } if (is_hardfork_) { @@ -260,28 +252,124 @@ void Collator::start_up() { if (!is_hardfork_) { LOG(DEBUG) << "sending get_external_messages() query to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, - [self = get_self()](td::Result, int>>> res) -> void { + auto token = perf_log_.start_action("get_external_messages"); + td::actor::send_closure_later( + manager, &ValidatorManager::get_external_messages, shard_, + [self = get_self(), + token = std::move(token)](td::Result, int>>> res) mutable -> void { LOG(DEBUG) << "got answer to get_external_messages() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res)); + td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res), + std::move(token)); }); } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages - LOG(DEBUG) << "sending get_shard_blocks() query to Manager"; + LOG(DEBUG) << "sending get_shard_blocks_for_collator() query to Manager"; ++pending; - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_blocks, prev_blocks[0], - [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_shard_blocks() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); - }); + auto token = perf_log_.start_action("get_shard_blocks_for_collator"); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], + [self = get_self(), token = std::move(token)]( + td::Result>> res) mutable -> void { + LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, + std::move(res), std::move(token)); + }); } - // 6. set timeout + // 6. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &Collator::after_get_storage_stat_cache, std::move(res), + std::move(token)); + }); + // 7. set timeout alarm_timestamp() = timeout; CHECK(pending); } +/** + * Load previous states and blocks from DB + */ +void Collator::load_prev_states_blocks() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + if (prev_blocks[i].seqno()) { + // 3.2. load block + LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); + } + } +} + +/** + * Write optimistic prev block as block data, load previous state to apply Merkle update to it + */ +bool Collator::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 3.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state_merge"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } + // 3.2. load block + LOG(DEBUG) << "use optimistic prev block " << prev_blocks[0].to_str(); + ++pending; + auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); + td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, + std::move(token)); + return true; +} + /** * Raises an error when timeout is reached. */ @@ -347,13 +435,25 @@ bool Collator::fatal_error(td::Status error) { if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; - run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, - attempt_idx_ + 1); + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_mc_block_id, + .prev = prev_blocks, + .is_hardfork = false, + .creator = created_by_, + .validator_set = validator_set_, + .collator_opts = collator_opts_, + .collator_node_id = collator_node_id_, + .skip_store_candidate = skip_store_candidate_, + .attempt_idx = attempt_idx_ + 1, + .optimistic_prev_block = optimistic_prev_block_}, + manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { + LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; + LOG(INFO) << perf_log_; + finalize_stats(); + stats_.status = error.clone(); + td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_)); main_promise(std::move(error)); - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, BlockIdExt{new_id, RootHash::zero(), FileHash::zero()}, - work_timer_.elapsed(), cpu_work_timer_.elapsed(), td::optional{}); } busy_ = false; } @@ -473,12 +573,14 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, - [self = get_self(), blkid](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, - blkid, std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno()); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, + [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -506,9 +608,11 @@ Ref Collator::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error("cannot load auxiliary masterchain state for "s + blkid.to_str() + " : " + res.move_as_error().to_string()); @@ -570,9 +674,11 @@ bool Collator::preprocess_prev_mc_state() { * * @param res The retrieved masterchain state. */ -void Collator::after_get_mc_state(td::Result, BlockIdExt>> res) { +void Collator::after_get_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_mc_state()"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -589,12 +695,14 @@ void Collator::after_get_mc_state(td::Result, Bl // NB. it is needed only for creating a correct ExtBlkRef reference to it, which requires start_lt and end_lt LOG(DEBUG) << "sending wait_block_data() query #-1 for " << mc_block_id_.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, - [self = get_self()](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #-1"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, - std::move(res)); - }); + auto token = perf_log_.start_action("wait_block_data #-1"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #-1"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, std::move(res), + std::move(token)); + }); } check_pending(); } @@ -605,9 +713,10 @@ void Collator::after_get_mc_state(td::Result, Bl * @param idx The index of the previous shard block (0 or 1). * @param res The retrieved shard state. */ -void Collator::after_get_shard_state(int idx, td::Result> res) { +void Collator::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_shard_state(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -636,11 +745,12 @@ void Collator::after_get_shard_state(int idx, td::Result> res) { * Callback function called after retrieving block data for a previous block. * * @param idx The index of the previous block (0 or 1). - * @param res The retreved block data. + * @param res The retrieved block data. */ -void Collator::after_get_block_data(int idx, td::Result> res) { +void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -664,6 +774,14 @@ void Collator::after_get_block_data(int idx, td::Result> res) { CHECK(!idx); prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); + } else { + Ref root = prev_block_data[idx]->root_cell(); + auto proof = create_block_state_proof(root); + if (proof.is_error()) { + fatal_error(proof.move_as_error()); + return; + } + block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } } check_pending(); @@ -674,8 +792,10 @@ void Collator::after_get_block_data(int idx, td::Result> res) { * * @param res The retrieved shard block descriptions. */ -void Collator::after_get_shard_blocks(td::Result>> res) { +void Collator::after_get_shard_blocks(td::Result>> res, + td::PerfLogAction token) { --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -686,6 +806,49 @@ void Collator::after_get_shard_blocks(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { + --pending; + token.finish(res); + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + check_pending(); +} + +/** + * Callback function called after retrieving previous state for optimistic prev block + * + * @param res The retrieved state. + */ +void Collator::after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token) { + LOG(DEBUG) << "in Collator::after_get_shard_state_optimistic()"; + token.finish(res); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -693,7 +856,7 @@ void Collator::after_get_shard_blocks(td::Resultset_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); + global_version_ = config_->get_global_version(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); @@ -714,6 +877,8 @@ bool Collator::unpack_last_mc_state() { store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); + full_collated_data_ = config_->has_capability(capFullCollatedData) || collator_opts_->force_full_collated_data; + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -733,15 +898,24 @@ bool Collator::unpack_last_mc_state() { LOG(INFO) << "Attempt #3: bytes, gas limits /= 2"; block_limits_->bytes.multiply_by(0.5); block_limits_->gas.multiply_by(0.5); + block_limits_->collated_data.multiply_by(0.5); } else if (attempt_idx_ == 4) { LOG(INFO) << "Attempt #4: bytes, gas limits /= 4"; block_limits_->bytes.multiply_by(0.25); block_limits_->gas.multiply_by(0.25); + block_limits_->collated_data.multiply_by(0.25); + } + if (collator_opts_->ignore_collated_data_limits) { + block_limits_->collated_data = block::ParamLimits{1 << 30, 1 << 30, 1 << 30}; } LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft() << ", " << block_limits_->bytes.hard() << "]"; LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", " << block_limits_->gas.hard() << "]"; + LOG(DEBUG) << "block limits: lt_delta [" << block_limits_->lt_delta.underload() << ", " + << block_limits_->lt_delta.soft() << ", " << block_limits_->lt_delta.hard() << "]"; + LOG(DEBUG) << "block limits: collated_data_bytes [" << block_limits_->collated_data.underload() << ", " + << block_limits_->collated_data.soft() << ", " << block_limits_->collated_data.hard() << "]"; if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { LOG(ERROR) << "block generation capabilities " << config_->get_capabilities() << " have been enabled in global configuration, but we support only " << supported_capabilities() @@ -797,6 +971,9 @@ bool Collator::request_neighbor_msg_queues() { auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return fatal_error(-667, "cannot obtain shard hash for neighbor "s + blk_id.to_str()); @@ -807,16 +984,24 @@ bool Collator::request_neighbor_msg_queues() { } neighbors_.emplace_back(*shard_ptr); } - int i = 0; + std::vector top_blocks; + unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &Collator::got_neighbor_out_queue, i, std::move(res)); - }); + if (prev_block_idx(descr.blk_) == -1) { + top_blocks.push_back(descr.blk_); + } ++i; } + ++pending; + auto token = perf_log_.start_action("neighbor_msg_queues"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_neighbor_msg_queue_proofs, shard_, std::move(top_blocks), timeout, + [self = get_self(), + token = std::move(token)](td::Result>> res) mutable { + td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res), + std::move(token)); + }); return true; } @@ -844,18 +1029,75 @@ bool Collator::request_out_msg_queue_size() { /** * Handles the result of obtaining the outbound queue for a neighbor. * - * @param i The index of the neighbor. - * @param res The obtained outbound queue. + * @param R The result of retrieving neighbor message queues (top block id -> queue). */ -void Collator::got_neighbor_out_queue(int i, td::Result> res) { +void Collator::got_neighbor_msg_queues(td::Result>> R, + td::PerfLogAction token) { --pending; - if (res.is_error()) { - fatal_error(res.move_as_error()); + double duration = token.finish(R); + if (R.is_error()) { + fatal_error(R.move_as_error_prefix("failed to get neighbor msg queues: ")); + return; + } + LOG(INFO) << "neighbor output queues fetched, took " << duration << "s"; + auto res = R.move_as_ok(); + unsigned i = 0; + stats_.neighbors.resize(neighbors_.size()); + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { + got_neighbor_msg_queue( + i, Ref{true, descr.blk_, prev_states[prev_idx]->root_cell(), td::Ref{}, true}); + } else { + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); + } + ++i; + } + check_pending(); +} + +void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { + BlockIdExt block_id = neighbors_.at(i).blk_; + if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { + block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); + } + + auto &neighbor_stats = stats_.neighbors.at(i); + neighbor_stats.shard = block_id.shard_full(); + neighbor_stats.is_trivial = shard_intersects(block_id.shard_full(), shard_); + neighbor_stats.is_local = res->is_local_; + neighbor_stats.msg_limit = res->msg_count_; + + Ref state_root; + if (block_id.is_masterchain()) { + state_root = res->state_root_; + } else { + neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); + state_root = neighbor_proof_builders_.back().root(); + if (full_collated_data_ && !block_id.is_masterchain()) { + neighbor_proof_builders_.back().set_cell_load_callback([&](const vm::LoadedCell& cell) { + on_cell_loaded(cell); + }); + } + } + auto state = ShardStateQ::fetch(block_id, {}, state_root); + if (state.is_error()) { + fatal_error(state.move_as_error()); + return; + } + auto outq_descr_res = state.move_as_ok()->message_queue(); + if (outq_descr_res.is_error()) { + fatal_error(outq_descr_res.move_as_error()); return; } - Ref outq_descr = res.move_as_ok(); + Ref outq_descr = outq_descr_res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); - LOG(WARNING) << "obtained outbound queue for neighbor #" << i << " : " << descr.shard().to_str(); + LOG(WARNING) << "obtained outbound queue for neighbor #" << i << "(" << descr.shard().to_str() << ")"; if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); fatal_error( @@ -871,7 +1113,12 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) fatal_error("cannot unpack neighbor output queue info"); return; } - descr.set_queue_root(qinfo.out_queue->prefetch_ref(0)); + auto queue_root = qinfo.out_queue->prefetch_ref(0); + descr.set_queue_root(queue_root); + if (res->msg_count_ != -1) { + LOG(INFO) << "neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; + neighbor_msg_queues_limits_[block_id.shard_full()] = res->msg_count_; + } // comment the next two lines in the future when the output queues become huge // CHECK(block::gen::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); // CHECK(block::tlb::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); @@ -889,22 +1136,16 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) return; } outq_descr.clear(); - do { - // require masterchain blocks referred to in ProcessedUpto - // TODO: perform this only if there are messages for this shard in our output queue - // .. (have to check the above condition and perform a `break` here) .. - // .. - for (const auto& entry : descr.processed_upto->list) { - Ref state; - if (!request_aux_mc_state(entry.mc_seqno, state)) { - return; - } + // require masterchain blocks referred to in ProcessedUpto + // TODO: perform this only if there are messages for this shard in our output queue + // .. (have to check the above condition and perform a `break` here) .. + // .. + for (const auto& entry : descr.processed_upto->list) { + Ref state; + if (!request_aux_mc_state(entry.mc_seqno, state)) { + return; } - } while (false); - if (!pending) { - LOG(INFO) << "all neighbor output queues fetched"; } - check_pending(); } /** @@ -931,7 +1172,7 @@ void Collator::got_out_queue_size(size_t i, td::Result res) { /** * Unpacks and merges the states of two previous blocks. * Used if the block is after_merge. - * + * * @returns True if the unpacking and merging was successful, false otherwise. */ bool Collator::unpack_merge_last_state() { @@ -946,6 +1187,11 @@ bool Collator::unpack_merge_last_state() { } // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { + on_cell_loaded(cell); + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // 2. extract back slightly virtualized roots of the two original states Ref root0, root1; @@ -990,6 +1236,11 @@ bool Collator::unpack_last_state() { prev_state_root_pure_ = prev_states.at(0)->root_cell(); // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { + on_cell_loaded(cell); + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // unpack previous state block::ShardState ss; @@ -1055,7 +1306,7 @@ bool Collator::split_last_state(block::ShardState& ss) { /** * Imports the shard state data into the Collator object. - * + * * SETS: account_dict = account_dict_estimator_, shard_libraries_, mc_state_extra * total_balance_ = old_total_balance_, total_validator_fees_ * SETS: overload_history_, underload_history_ @@ -1081,9 +1332,11 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { total_validator_fees_ = std::move(ss.total_validator_fees_); old_global_balance_ = std::move(ss.global_balance_); out_msg_queue_ = std::move(ss.out_msg_queue_); + old_out_msg_queue_ = std::make_unique(*out_msg_queue_); processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); dispatch_queue_ = std::move(ss.dispatch_queue_); + old_dispatch_queue_ = std::make_unique(*dispatch_queue_); block_create_stats_ = std::move(ss.block_create_stats_); if (ss.out_msg_queue_size_) { have_out_msg_queue_size_in_state_ = true; @@ -1120,6 +1373,7 @@ bool Collator::add_trivial_neighbor_after_merge() { nb.set_queue_root(out_msg_queue_->get_root_cell()); nb.processed_upto = processed_upto_; nb.blk_.id.shard = get_shard(); + stats_.neighbors[i].shard = nb.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb.blk_.to_str() << " with shard expansion (immediate after-merge adjustment)"; } else { @@ -1195,10 +1449,13 @@ bool Collator::add_trivial_neighbor() { CHECK(sibling_out_msg_queue_); CHECK(sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); nb2.set_queue_root(sibling_out_msg_queue_->get_root_cell()); nb2.processed_upto = sibling_processed_upto_; nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (immediate after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -1218,6 +1475,8 @@ bool Collator::add_trivial_neighbor() { CHECK(!sibling_out_msg_queue_); CHECK(!sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); auto sib_shard = ton::shard_sibling(shard_); // compute the part of virtual sibling's OutMsgQueue with destinations in our shard @@ -1237,6 +1496,7 @@ bool Collator::add_trivial_neighbor() { return fatal_error("error splitting ProcessedUpto for our virtual sibling"); } nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (continued after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -1321,7 +1581,7 @@ bool Collator::check_prev_block_exact(const BlockIdExt& listed, const BlockIdExt /** * Checks the validity of the shard configuration of the current shard. - * + * * @returns True if the shard configuration is valid, false otherwise. */ bool Collator::check_this_shard_mc_info() { @@ -1471,6 +1731,7 @@ bool Collator::init_block_limits() { } block_limits_->usage_tree = state_usage_tree_.get(); block_limit_status_ = std::make_unique(*block_limits_); + block_limit_status_->collated_data_size_estimate = collated_data_stat.estimate_proof_size(); return true; } @@ -1696,8 +1957,8 @@ bool Collator::import_new_shard_top_blocks() { prev_descr.clear(); descr.clear(); } else { - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " - << prev_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " + << prev_bd->block_id().to_str(); CHECK(ures.move_as_ok()); store_shard_fees(std::move(prev_descr)); store_shard_fees(std::move(descr)); @@ -1739,7 +2000,7 @@ bool Collator::import_new_shard_top_blocks() { store_shard_fees(std::move(descr)); register_shard_block_creators(sh_bd->get_creator_list(chain_len)); shards_max_end_lt_ = std::max(shards_max_end_lt_, end_lt); - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str(); CHECK(ures.move_as_ok()); ++tb_act; used_shard_block_descr_.emplace_back(sh_bd); @@ -1800,10 +2061,8 @@ bool Collator::register_shard_block_creators(std::vector creator_li */ bool Collator::try_collate() { work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; @@ -1820,7 +2079,7 @@ bool Collator::try_collate() { last_proc_int_msg_.second.set_zero(); first_unproc_int_msg_.first = ~0ULL; first_unproc_int_msg_.second.set_ones(); - old_out_msg_queue_size_ = out_msg_queue_size_; + stats_.old_out_msg_queue_size = old_out_msg_queue_size_ = out_msg_queue_size_; if (is_masterchain()) { LOG(DEBUG) << "getting the list of special smart contracts"; auto res = config_->get_special_smartcontracts(); @@ -1906,7 +2165,7 @@ bool Collator::fix_processed_upto(block::MsgProcessedUptoCollection& upto) { /** * Initializes the unix time for the new block. - * + * * Unix time is set based on the current time, and the timestamps of the previous blocks. * If the previous block has a timestamp too far in the past then skipping importing external messages and new shard blocks is allowed. * @@ -2066,6 +2325,17 @@ bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { return true; } +bool Collator::create_output_queue_merger() { + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + auto it = neighbor_msg_queues_limits_.find(descr.shard()); + td::int32 msg_limit = it == neighbor_msg_queues_limits_.end() ? -1 : it->second; + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + } + nb_out_msgs_ = std::make_unique(shard_, neighbor_queues); + return true; +} + /** * Initializes value_flow_ and computes fees for creating the new block. * @@ -2107,14 +2377,20 @@ bool Collator::init_value_create() { bool Collator::do_collate() { // After do_collate started it will not be interrupted by timeout alarm_timestamp() = td::Timestamp::never(); + auto token = perf_log_.start_action("do_collate"); + td::Status status = td::Status::Error("some error"); + SCOPE_EXIT { + token.finish(status); + }; LOG(WARNING) << "do_collate() : start"; if (!fetch_config_params()) { return fatal_error("cannot fetch required configuration parameters from masterchain state"); } LOG(DEBUG) << "config parameters fetched, creating message dictionaries"; - in_msg_dict = std::make_unique(256, block::tlb::aug_InMsgDescr); - out_msg_dict = std::make_unique(256, block::tlb::aug_OutMsgDescr); + aug_InMsgDescr.global_version = aug_OutMsgDescr.global_version = global_version_; + in_msg_dict = std::make_unique(256, aug_InMsgDescr); + out_msg_dict = std::make_unique(256, aug_OutMsgDescr); LOG(DEBUG) << "message dictionaries created"; if (max_lt == start_lt) { ++max_lt; @@ -2132,7 +2408,9 @@ bool Collator::do_collate() { // 1.3. create OutputQueueMerger from adjusted neighbors CHECK(!nb_out_msgs_); LOG(DEBUG) << "creating OutputQueueMerger"; - nb_out_msgs_ = std::make_unique(shard_, neighbors_); + if (!create_output_queue_merger()) { + return fatal_error("cannot compute the value to be created / minted / recovered"); + } // 1.4. compute created / minted / recovered if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); @@ -2230,6 +2508,7 @@ bool Collator::do_collate() { if (!create_block_candidate()) { return fatal_error("cannot serialize a new Block candidate"); } + status = td::Status::OK(); return true; } @@ -2262,12 +2541,17 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli /** * Cleans up the outbound message queue by removing messages that have already been imported by neighbors. - * + * * Cleanup may be interrupted early if it takes too long. * * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + stats_.work_time.queue_cleanup = timer.elapsed_both(); + }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { FLOG(INFO) { @@ -2287,8 +2571,21 @@ bool Collator::out_msg_queue_cleanup() { << nb.blk_.to_str()); } } + auto queue_root = out_msg_queue_->get_root_cell(); + if (queue_root.is_null()) { + LOG(DEBUG) << "out_msg_queue is empty"; + return true; + } + // Unwrap UsageCell: don't build proof for visiting output queue (unless something is deleted) + auto r_cell = queue_root->load_cell(); + if (r_cell.is_error()) { + return fatal_error(r_cell.move_as_error()); + } + auto pure_out_msg_queue = + std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); td::uint32 deleted = 0; - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + bool ok = pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { + vm::CellSlice& cs = value.write(); assert(n == 352); block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; @@ -2297,7 +2594,7 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; + return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; @@ -2318,10 +2615,19 @@ bool Collator::out_msg_queue_cleanup() { --out_msg_queue_size_; LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + // Get value from out_msg_queue_ instead of pure_out_msg_queue (for proof) + auto value2 = out_msg_queue_->lookup_delete_with_extra(key, n); + CHECK(value2.not_null()); + vm::CellSlice& cs2 = value2.write(); + CHECK(cs2.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs2) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - return -1; + return false; } register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { @@ -2329,11 +2635,12 @@ bool Collator::out_msg_queue_cleanup() { block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } } - return !delivered; + return true; }); + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; - if (res < 0) { + if (!ok) { return fatal_error("error scanning/updating OutMsgQueue"); } } else { @@ -2411,6 +2718,7 @@ bool Collator::out_msg_queue_cleanup() { std::swap(queue_parts[i], queue_parts.back()); queue_parts.pop_back(); } + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue, remaining queue size is " << out_msg_queue_size_; } @@ -2448,9 +2756,92 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr return nullptr; } ptr->block_lt = start_lt; + if (!init_account_storage_dict(*ptr)) { + return nullptr; + } return ptr; } +/** + * If full collated data is enabled, initialize account storage dict and prepare MerkleProofBuilder for it + * + * @param account Account to initialize storage dict for + * @return True on success, False on failure + */ +bool Collator::init_account_storage_dict(block::Account& account) { + if (!account.storage_dict_hash || account.storage.is_null()) { + return true; + } + td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); + if (storage_dict_hash.is_zero()) { + return true; + } + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.prelim_storage_stat = timer.elapsed_both(); + }; + td::Ref cached_dict_root = + storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; + if (cached_dict_root.not_null()) { + CHECK(td::Bits256{cached_dict_root->get_hash().bits()} == storage_dict_hash); + LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" + << account.storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += account.storage_used.cells; + } else if (account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += account.storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += account.storage_used.cells; + } + if (!full_collated_data_ || is_masterchain()) { + if (cached_dict_root.not_null()) { + auto S = account.init_account_storage_stat(cached_dict_root); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << account.addr.to_hex() << ": ")); + } + } + return true; + } + + AccountStorageDict& dict = account_storage_dicts_[storage_dict_hash]; + if (!dict.inited) { + dict.inited = true; + td::Ref dict_root; + if (cached_dict_root.not_null()) { + dict_root = cached_dict_root; + } else { + // don't mark cells in account state as loaded during compute_account_storage_dict + state_usage_tree_->set_ignore_loads(true); + auto res = account.compute_account_storage_dict(); + state_usage_tree_->set_ignore_loads(false); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + dict_root = res.move_as_ok(); + if (dict_root.is_null()) { // Impossible if storage_dict_hash is not zero + return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() + << ": dict is empty"); + } + } + dict.mpb = vm::MerkleProofBuilder(std::move(dict_root)); + dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { + on_cell_loaded(cell); + }); + } + auto S = account.init_account_storage_stat(dict.mpb.root()); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + return true; +} + + /** * Looks up an account in the Collator's account map. * @@ -2500,6 +2891,184 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Removes UsageCell's from new_root's subtree, replacing them with regular DataCell's + * + * @param old_root Original root (not wrapped in UsageCell) + * @param new_root Cell to remove UsageCell's from + * @param usage_tree CellUsageTree for old_root + * + * @returns new_root without UsageCell's + */ +static td::Ref clean_usage_cells(td::Ref old_root, td::Ref new_root, + const vm::CellUsageTree& usage_tree) { + if (new_root.is_null()) { + return {}; + } + + td::HashMap> visited_cells; + std::function, vm::CellUsageTree::NodeId)> dfs_old = [&](td::Ref cell, + vm::CellUsageTree::NodeId node_id) { + visited_cells[cell->get_hash()] = cell; + if (!usage_tree.is_loaded(node_id)) { + return; + } + vm::CellSlice cs{vm::NoVm(), cell}; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs_old(cs.prefetch_ref(i), usage_tree.get_child(node_id, i)); + } + }; + if (old_root.not_null()) { + dfs_old(old_root, usage_tree.root_id()); + } + + std::function(td::Ref)> dfs = [&](td::Ref cell) -> td::Ref { + auto it = visited_cells.find(cell->get_hash()); + if (it != visited_cells.end()) { + return it->second; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + CHECK(loaded_cell.virt.get_virtualization() == 0); + td::Ref data_cell = std::move(loaded_cell.data_cell); + td::Ref children[vm::Cell::max_refs]; + bool changed = false; + for (unsigned i = 0; i < data_cell->size_refs(); ++i) { + td::Ref child = data_cell->get_ref(i); + children[i] = dfs(child); + if (children[i] != child) { + changed = true; + } + } + if (changed) { + data_cell = vm::DataCell::create(td::Slice{data_cell->get_data(), (data_cell->size() + 7) / 8}, data_cell->size(), + {children, data_cell->size_refs()}, data_cell->is_special()) + .move_as_ok(); + CHECK(data_cell->get_hash() == cell->get_hash()); + } + return visited_cells[cell->get_hash()] = data_cell; + }; + return dfs(new_root); +} + +/** + * Decides whether to include storage dict proof to collated data for this account or not. + * + * @param account Account object + * + * @returns True if the operation is successful, false otherwise. + */ +bool Collator::process_account_storage_dict(block::Account& account) { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.final_storage_stat += timer.elapsed_both(); + }; + bool store_dict_to_cache = account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS; + if (!account.orig_storage_dict_hash) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } + return true; + } + td::Bits256 storage_dict_hash = account.orig_storage_dict_hash.value(); + auto it = account_storage_dicts_.find(storage_dict_hash); + if (it == account_storage_dicts_.end()) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } + return true; + } + CHECK(full_collated_data_ && !is_masterchain()); + AccountStorageDict& dict = it->second; + td::Ref original_dict_root = dict.mpb.original_root(); + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + dict_root = clean_usage_cells(original_dict_root, dict_root, dict.mpb.get_usage_tree()); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } + if (dict.add_to_collated_data) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; + return true; + } + if (dict.storage_stat_updates.empty()) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : not required (no storage updates)"; + return true; + } + + td::HashSet visited; + bool calculate_proof_size_diff = true; + td::int64 proof_size_diff = 0; + vm::Dictionary original_dict{original_dict_root, 256}; + std::function&)> dfs = [&](const Ref& cell) -> bool { + if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { + return true; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + if (original_dict.lookup(cell->get_hash().bits(), 256).not_null()) { + if (calculate_proof_size_diff) { + switch (collated_data_stat.get_cell_status(cell->get_hash())) { + case vm::ProofStorageStat::c_none: + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_prunned: + proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_loaded: + break; + } + if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { + return false; + } + } else { + collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); + } + } + vm::CellSlice cs{std::move(loaded_cell.data_cell)}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + if (!dfs(cs.prefetch_ref(i))) { + return false; + } + } + return true; + }; + + // Visit cells that were used in storage stat computation to calculate collated data increase + bool account_proof_too_big = false; + state_usage_tree_->set_ignore_loads(true); + for (const auto& cell : dict.storage_stat_updates) { + if (!dfs(cell)) { + account_proof_too_big = true; + break; + } + } + state_usage_tree_->set_ignore_loads(false); + + if (account_proof_too_big) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size>=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", include dict in collated data"; + dict.add_to_collated_data = true; + collated_data_stat.add_loaded_cells(dict.proof_stat); + } else { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() + << ", DO NOT include dict in collated data"; + // Include account storage in collated data + calculate_proof_size_diff = false; + visited.clear(); + for (const auto& cell : dict.storage_stat_updates) { + CHECK(dfs(cell)); + } + } + + return true; +} + /** * Combines account transactions and updates the ShardAccountBlocks and ShardAccounts. * @@ -2589,6 +3158,9 @@ bool Collator::combine_account_transactions() { } } } + if (!process_account_storage_dict(acc)) { + return false; + } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -2661,7 +3233,7 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< && cb.store_long_bool(0x4ff, 11) // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 && cb.store_bits_bool(addr) // address:bits256 => dest:MsgAddressInt && amount.store(cb) // value:CurrencyCollection - && cb.store_zeroes_bool(4 + 4) // ihr_fee:Grams fwd_fee:Grams + && cb.store_zeroes_bool(4 + 4) // extra_flags:(VarUInteger 16) fwd_fee:Grams && cb.store_long_bool(lt, 64) // created_lt:uint64 && cb.store_long_bool(now_, 32) // created_at:uint32 && cb.store_zeroes_bool(2) // init:(Maybe ...) body:(Either X ^X) = Message X @@ -2727,9 +3299,16 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t << "last transaction time in the state of account " << workchain() << ":" << smc_addr.to_hex() << " is too large")); } + set_current_tx_storage_dict(*acc); std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trans->time_tvm; + stats_.work_time.trx_storage_stat += trans->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + }; if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -2761,9 +3340,11 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t if (!update_account_dict_estimation(*trans)) { return fatal_error(-666, "cannot update account dict size estimation"); } + update_account_storage_dict_info(*trans); update_max_lt(acc->last_trans_end_lt_); block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; register_new_msgs(*trans, std::move(new_msg_metadata)); + ++stats_.transactions; return true; } @@ -2830,8 +3411,9 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, if (it != last_dispatch_queue_emitted_lt_.end()) { after_lt = std::max(after_lt, it->second); } + set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt, &stats_); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -2858,6 +3440,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, fatal_error("cannot update account dict size estimation"); return {}; } + update_account_storage_dict_info(*trans); td::optional new_msg_metadata; if (external || is_special_tx) { @@ -2869,6 +3452,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, register_new_msgs(*trans, std::move(new_msg_metadata)); update_max_lt(acc->last_trans_end_lt_); value_flow_.burned += trans->blackhole_burned; + ++stats_.transactions; return trans_root; } @@ -2885,19 +3469,17 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. + * @param collation_stats Stats to write real/cpu time to (optional) * * @returns A Result object containing the created transaction. * Returns error_code == 669 if the error is fatal and the block can not be produced. * Returns error_code == 701 if the transaction can not be included into block, but it's ok (external or too early internal). */ -td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - block::SerializeConfig* serialize_cfg, - bool external, LogicalTime after_lt) { +td::Result> Collator::impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt, CollationStats* stats) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large"); @@ -2909,64 +3491,74 @@ td::Result> Collator::impl_crea std::unique_ptr trans = std::make_unique( *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { - if (external) { - // inbound external message was not accepted - return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution"); + { + td::RealCpuTimer timer; + SCOPE_EXIT { + if (stats) { + stats->work_time.trx_tvm += trans->time_tvm; + stats->work_time.trx_storage_stat += trans->time_storage_stat; + stats->work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + } + }; + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669, "cannot unpack input message for a new transaction"); } - return td::Status::Error(-669, "cannot unpack input message for a new transaction"); - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } } - if (!external && !trans->prepare_credit_phase()) { + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + auto const& cp = *trans->compute_phase; + return td::Status::Error( + -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" + << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used + << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } } - } else { - if (!external && !trans->prepare_credit_phase()) { + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && + !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - } - if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return td::Status::Error( - -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - auto const& cp = *trans->compute_phase; - return td::Status::Error( - -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" - << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used - << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)"); + if (!trans->serialize(*serialize_cfg)) { + return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } } - if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (trans->bounce_enabled && - (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && - !trans->prepare_bounce_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->serialize(*serialize_cfg)) { - return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); - } return std::move(trans); } @@ -3007,12 +3599,12 @@ bool Collator::update_last_proc_int_msg(const std::pairtick * 2 + found->tock : config_->get_smc_tick_tock(smc_addr.cbits())); @@ -3370,19 +3962,7 @@ bool Collator::delete_out_msg_queue_msg(td::ConstBitPtr key) { return register_out_msg_queue_op(); } -/** - * Processes an inbound message from a neighbor's outbound queue. - * The message may create a transaction or be enqueued. - * - * @param enq_msg The inbound message serialized using EnqueuedMsg TLB-scheme. - * @param lt The logical time of the message. - * @param key The 32+64+256-bit key of the message. - * @param src_nb The description of the source neighbor shard. - * - * @returns True if the message was processed successfully, false otherwise. - */ -bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb) { +bool Collator::precheck_inbound_message(Ref enq_msg, ton::LogicalTime lt) { ton::LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG @@ -3411,6 +3991,25 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to hand-written checks"; return false; } + return true; +} + +/** + * Processes an inbound message from a neighbor's outbound queue. + * The message may create a transaction or be enqueued. + * + * @param enq_msg The inbound message serialized using EnqueuedMsg TLB-scheme. + * @param lt The logical time of the message. + * @param key The 32+64+256-bit key of the message. + * @param src_nb_idx The index of the source neighbor shard. + * + * @returns True if the message was processed successfully, false otherwise. + */ +bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, + int src_nb_idx) { + const auto& src_nb = neighbors_.at(src_nb_idx); + ton::LogicalTime enqueued_lt = enq_msg->prefetch_ulong(64); + auto msg_env = enq_msg->prefetch_ref(); // 1. unpack MsgEnvelope block::tlb::MsgEnvelope::Record_std env; if (!tlb::unpack_cell(msg_env, env)) { @@ -3506,6 +4105,8 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT << " enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already processed by us before, skipping"; // should we dequeue the message if it is ours (after a merge?) // (it should have been dequeued by out_msg_queue_cleanup() before) + auto &neighbor_stats = stats_.neighbors.at(src_nb_idx); + ++neighbor_stats.skipped_msgs; return true; } // 6.1. check whether we have already processed this message by IHR @@ -3578,6 +4179,10 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { return PSTRING() << "block_full lt_delta " << lt_delta; } + auto collated_data_bytes = block_limit_status.collated_data_size_estimate; + if (!block_limit_status.limits.collated_data.fits(cls, collated_data_bytes)) { + return PSTRING() << "block_full collated_data " << collated_data_bytes; + } return ""; } @@ -3588,11 +4193,33 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_internal_messages() { - if (have_unprocessed_account_dispatch_queue_) { - return true; - } - while (!block_full_ && !nb_out_msgs_->is_eof()) { + SCOPE_EXIT { + stats_.load_fraction_internals = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; + while (!nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); + auto kv = nb_out_msgs_->extract_cur(); + CHECK(kv && kv->msg.not_null()); + auto &neighbor_stats = stats_.neighbors.at(kv->source); + if (kv->limit_exceeded) { + LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; + neighbor_stats.limit_reached = true; + block::EnqueuedMsgDescr enq; + enq.unpack(kv->msg.write()); // Visit cells to include it in proof + break; + } + if (!precheck_inbound_message(kv->msg, kv->lt)) { + if (verbosity > 1) { + std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + } + return fatal_error("error processing inbound internal message"); + } + if (have_unprocessed_account_dispatch_queue_) { + LOG(INFO) << "have unprocessed account dispatch queue, stop processing inbound internal messages"; + return true; + } if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); @@ -3609,17 +4236,16 @@ bool Collator::process_inbound_internal_messages() { if (!check_cancelled()) { return false; } - auto kv = nb_out_msgs_->extract_cur(); - CHECK(kv && kv->msg.not_null()); LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; + ++neighbor_stats.processed_msgs; if (verbosity > 2) { FLOG(INFO) { sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; block::gen::t_EnqueuedMsg.print(sb, kv->msg); }; } - if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { + if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), kv->source)) { if (verbosity > 1) { FLOG(INFO) { sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() @@ -3638,10 +4264,13 @@ bool Collator::process_inbound_internal_messages() { /** * Processes inbound external messages. * Messages are processed until the soft limit is reached, medium timeout is reached or there are no more messages. - * + * * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_external_messages() { + SCOPE_EXIT { + stats_.load_fraction_externals = block_limit_status_->load_fraction(block::ParamLimits::cl_soft); + }; if (skip_extmsg_) { LOG(INFO) << "skipping processing of inbound external messages"; return true; @@ -3761,6 +4390,9 @@ int Collator::process_external_message(Ref msg) { * @returns True if the processing was successful, false otherwise. */ bool Collator::process_dispatch_queue() { + SCOPE_EXIT { + stats_.load_fraction_dispatch = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; if (out_msg_queue_size_ > defer_out_queue_size_limit_ && old_out_msg_queue_size_ > hard_defer_out_queue_size_limit_) { return true; } @@ -4220,6 +4852,9 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema * @returns True if all new messages were processed successfully, false otherwise. */ bool Collator::process_new_messages(bool enqueue_only) { + SCOPE_EXIT { + stats_.load_fraction_new_msgs = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; while (!new_msgs.empty()) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); @@ -4828,7 +5463,8 @@ bool Collator::check_block_overload() { block_size_estimate_ = block_limit_status_->estimate_block_size(); LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt - << " size_estimate=" << block_size_estimate_; + << " size_estimate=" << block_size_estimate_ + << " collated_size_estimate=" << block_limit_status_->collated_data_size_estimate; block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); if (block_limit_class_ >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { std::string message = "block is overloaded "; @@ -5123,6 +5759,26 @@ bool Collator::update_account_dict_estimation(const block::transaction::Transact return true; } +/** + * Update `storage_stat_updates` for AccountStorageDict for the new transaction. + * + * @param trans Newly-created transaction. + */ +void Collator::update_account_storage_dict_info(const block::transaction::Transaction& trans) { + if (!trans.account.orig_storage_dict_hash) { + return; + } + auto it = account_storage_dicts_.find(trans.account.orig_storage_dict_hash.value()); + if (it == account_storage_dicts_.end()) { + return; + } + for (const Ref& cell : trans.storage_stat_updates) { + if (cell.not_null()) { + it->second.storage_stat_updates.push_back(cell); + } + } +} + /** * Creates a new shard state and the Merkle update. * @@ -5491,6 +6147,10 @@ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { * @returns True if the new block is successfully created, false otherwise. */ bool Collator::create_block() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block += timer.elapsed_both(); + }; Ref block_info, extra; if (!create_block_info(block_info)) { return fatal_error("cannot create BlockInfo for the new block"); @@ -5565,23 +6225,155 @@ Ref Collator::collate_shard_block_descr_set() { return cell; } +/** + * Visits certain cells in out msg queue and dispatch queue to add them to the proof + * + * @returns True on success, False if error occurred + */ +bool Collator::prepare_msg_queue_proof() { + auto res = old_out_msg_queue_->scan_diff( + *out_msg_queue_, + [this](td::ConstBitPtr key, int key_len, Ref old_value, Ref new_value) { + old_value = old_out_msg_queue_->extract_value(std::move(old_value)); + new_value = out_msg_queue_->extract_value(std::move(new_value)); + if (new_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + } + if (old_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + } + return true; + }, + 2); + if (!res) { + return false; + } + res = old_dispatch_queue_->scan_diff( + *dispatch_queue_, + [this](td::ConstBitPtr, int, Ref old_value, Ref new_value) { + if (old_value.not_null()) { + old_value = old_dispatch_queue_->extract_value(std::move(old_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(old_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> max_lt; + CHECK(dispatch_dict.get_minmax_key(max_lt, true).not_null()); + } + if (new_value.not_null()) { + new_value = dispatch_queue_->extract_value(std::move(new_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(new_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> min_lt; + CHECK(dispatch_dict.get_minmax_key(min_lt, false).not_null()); + } + return true; + }, + 2); + return res; +} + /** * Creates collated data for the block. * * @returns True if the collated data was successfully created, false otherwise. */ bool Collator::create_collated_data() { - // TODO: store something into collated_roots_ + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_collated_data += timer.elapsed_both(); + }; // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); if (cell.is_null()) { return true; - return fatal_error("cannot collate the collection of used shard block descriptions"); + // return fatal_error("cannot collate the collection of used shard block descriptions"); } collated_roots_.push_back(std::move(cell)); } - // 2. ... + if (!full_collated_data_) { + return true; + } + // 2. Proofs for hashes of states: previous states + neighbors + for (const auto& p : block_state_proofs_) { + collated_roots_.push_back(p.second); + } + // 3. Previous state proof (only shadchains) + std::map> proofs; + if (!is_masterchain()) { + if (!prepare_msg_queue_proof()) { + return fatal_error("cannot prepare message queue proof"); + } + + state_usage_tree_->set_use_mark_for_is_loaded(false); + Ref state_proof = vm::MerkleProof::generate(prev_state_root_, [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); + if (state_proof.is_null()) { + return fatal_error("cannot generate Merkle proof for previous state"); + } + if (after_merge_) { + bool special; + auto cs = vm::load_cell_slice_special(state_proof, special); + CHECK(cs.special_type() == vm::CellTraits::SpecialType::MerkleProof); + cs = vm::load_cell_slice(cs.prefetch_ref(0)); + CHECK(cs.size_refs() == 2); + CHECK(cs.size() == 32); + CHECK(cs.prefetch_ulong(32) == 0x5f327da5U); + proofs[cs.prefetch_ref(0)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(0)); + proofs[cs.prefetch_ref(1)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(1)); + } else { + proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + } + } + // 4. Proofs for message queues + for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { + Ref proof = vm::MerkleProof::generate(mpb.original_root(), [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } + auto it = proofs.emplace(mpb.root()->get_hash().bits(), proof); + if (!it.second) { + it.first->second = vm::MerkleProof::combine(it.first->second, std::move(proof)); + if (it.first->second.is_null()) { + return fatal_error("cannot combine merkle proofs"); + } + } + } + + for (auto& p : proofs) { + collated_roots_.push_back(std::move(p.second)); + } + + // 5. Proofs for account storage dicts + for (auto& [_, dict] : account_storage_dicts_) { + if (!dict.add_to_collated_data) { + continue; + } + Ref proof = vm::MerkleProof::generate( + dict.mpb.original_root(), [&](const Ref& c) { return !collated_data_stat.is_loaded(c->get_hash()); }); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + collated_roots_.push_back(vm::CellBuilder() + .store_long(block::gen::AccountStorageDictProof::cons_tag[0], 32) + .store_ref(proof) + .finalize_novm()); + } return true; } @@ -5598,6 +6390,11 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block_candidate += timer.elapsed_both(); + }; + auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; vm::BagOfCells boc; @@ -5623,7 +6420,8 @@ bool Collator::create_block_candidate() { if (res.is_error()) { return fatal_error(res.move_as_error()); } - auto cdata_res = boc_collated.serialize_to_slice(31); + int cdata_serialize_mode = consensus_config.proto_version >= 5 ? 2 : 31; + auto cdata_res = boc_collated.serialize_to_slice(cdata_serialize_mode); if (cdata_res.is_error()) { LOG(ERROR) << "cannot serialize collated data"; return fatal_error(cdata_res.move_as_error()); @@ -5631,41 +6429,82 @@ bool Collator::create_block_candidate() { cdata_slice = cdata_res.move_as_ok(); } LOG(INFO) << "serialized block size " << blk_slice.size() << " bytes (preliminary estimate was " - << block_size_estimate_ << "), collated data " << cdata_slice.size() << " bytes"; + << block_size_estimate_ << ")"; auto st = block_limit_status_->st_stat.get_total_stat(); LOG(INFO) << "size regression stats: " << blk_slice.size() << " " << st.cells << " " << st.bits << " " << st.internal_refs << " " << st.external_refs << " " << block_limit_status_->accounts << " " << block_limit_status_->transactions; + LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " + << block_limit_status_->collated_data_size_estimate << ")"; + auto new_block_id_ext = ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), + block::compute_file_hash(blk_slice.as_slice())}; // 3. create a BlockCandidate - block_candidate = std::make_unique( - created_by_, - ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), - block::compute_file_hash(blk_slice.as_slice())}, - block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); + block_candidate = + std::make_unique(created_by_, new_block_id_ext, block::compute_file_hash(cdata_slice.as_slice()), + blk_slice.clone(), cdata_slice.clone()); + const bool need_out_msg_queue_broadcasts = !is_masterchain(); + if (need_out_msg_queue_broadcasts) { + // we can't generate two proofs at the same time for the same root (it is not currently supported by cells) + // so we have can't reuse new state and have to regenerate it with merkle update + auto new_state = vm::MerkleUpdate::apply(prev_state_root_pure_, state_update); + CHECK(new_state.not_null()); + CHECK(new_state->get_hash() == state_root->get_hash()); + CHECK(shard_conf_); + auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); + LOG(INFO) << "Build OutMsgQueueProofs for " << neighbor_list.size() << " neighbours"; + for (BlockId blk_id : neighbor_list) { + auto prefix = blk_id.shard_full(); + if (shard_intersects(prefix, shard_)) { + continue; + } + auto limits = mc_state_->get_imported_msg_queue_limits(blk_id.workchain); + + // one could use monitor_min_split_depth here, to decrease number of broadcasts + // but current implementation OutMsgQueueImporter doesn't support it + + auto r_proof = OutMsgQueueProof::build( + prefix, + {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, + limits); + if (r_proof.is_ok()) { + auto proof = r_proof.move_as_ok(); + CHECK(proof->msg_counts_.size() == 1); + block_candidate->out_msg_queue_proof_broadcasts.push_back(td::Ref( + true, OutMsgQueueProofBroadcast(prefix, new_block_id_ext, limits.max_bytes, limits.max_msgs, + std::move(proof->queue_proofs_), std::move(proof->block_state_proofs_), + proof->msg_counts_[0]))); + } else { + LOG(ERROR) << "Failed to build OutMsgQueueProof to " << prefix.to_str() << ": " << r_proof.error(); + } + } + } + // 3.1 check block and collated data size - auto consensus_config = config_->get_consensus_config(); if (block_candidate->data.size() > consensus_config.max_block_size) { return fatal_error(PSTRING() << "block size (" << block_candidate->data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_block_size << ")"); } - if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) { + if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size && + !collator_opts_->ignore_collated_data_limits) { return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - if (mode_ & CollateMode::skip_store_candidate) { - td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + if (skip_store_candidate_) { + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit(), td::PerfLogAction{}); } else { LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later( - manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); - }); + auto token = perf_log_.start_action("set_block_candidate"); + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), + [self = get_self(), token = std::move(token)](td::Result saved) mutable { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved), std::move(token)); + }); } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { @@ -5673,18 +6512,10 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } - - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); - LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - stats_.bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); - stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); - stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, - cpu_work_time, std::move(stats_)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } @@ -5693,8 +6524,9 @@ bool Collator::create_block_candidate() { * * @param saved The result of saving the block candidate to the disk. */ -void Collator::return_block_candidate(td::Result saved) { +void Collator::return_block_candidate(td::Result saved, td::PerfLogAction token) { // 6. return data to the original "caller" + token.finish(saved); if (saved.is_error()) { auto err = saved.move_as_error(); LOG(ERROR) << "cannot save block candidate: " << err.to_string(); @@ -5702,6 +6534,11 @@ void Collator::return_block_candidate(td::Result saved) { } else { CHECK(block_candidate); LOG(WARNING) << "sending new BlockCandidate to Promise"; + LOG(WARNING) << "collation took " << perf_timer_.elapsed() << " s"; + LOG(WARNING) << perf_log_; + finalize_stats(); + stats_.status = td::Status::OK(); + td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_)); main_promise(block_candidate->clone()); busy_ = false; stop(); @@ -5723,7 +6560,7 @@ void Collator::return_block_candidate(td::Result saved) { * @returns Result indicating the success or failure of the registration. * - If the external message is invalid, returns an error. * - If the external message has been previously rejected, returns an error - * - If the external message has been previuosly registered and accepted, returns false. + * - If the external message has been previously registered and accepted, returns false. * - Otherwise returns true. */ td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, @@ -5782,9 +6619,11 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, * * @param res The result of the external message retrieval operation. */ -void Collator::after_get_external_messages(td::Result, int>>> res) { +void Collator::after_get_external_messages(td::Result, int>>> res, + td::PerfLogAction token) { // res: pair {ext msg, priority} --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -5828,6 +6667,90 @@ td::uint32 Collator::get_skip_externals_queue_size() { return SKIP_EXTERNALS_QUEUE_SIZE; } +void Collator::finalize_stats() { + auto work_time = work_timer_.elapsed_both(); + LOG(WARNING) << "Collate query work time = " << work_time.real << "s, cpu time = " << work_time.cpu << "s"; + if (block_candidate) { + stats_.block_id = block_candidate->id; + stats_.collated_data_hash = block_candidate->collated_file_hash; + stats_.actual_bytes = (td::uint32)block_candidate->data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate->collated_data.size(); + } else { + stats_.block_id.id = new_id; + } + stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; + stats_.collated_at = td::Clocks::system(); + stats_.attempt = attempt_idx_; + stats_.is_validator = collator_node_id_.is_zero(); + stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() + : collator_node_id_.pubkey_hash(); + if (block_limit_status_) { + stats_.estimated_bytes = (td::uint32)block_limit_status_->estimate_block_size(); + stats_.gas = (td::uint32)block_limit_status_->gas_used; + stats_.lt_delta = (td::uint32)(block_limit_status_->cur_lt - block_limit_status_->limits.start_lt); + stats_.estimated_collated_data_bytes = (td::uint32)block_limit_status_->collated_data_size_estimate; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + stats_.cat_collated_data_bytes = + block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + } + stats_.total_time = perf_timer_.elapsed(); + stats_.work_time.total = work_time; + stats_.time_stats = (PSTRING() << perf_log_); + if (is_masterchain() && shard_conf_) { + shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { + stats_.shard_configuration.push_back(shard.top_block_id()); + return 0; + }); + } + stats_.new_out_msg_queue_size = out_msg_queue_size_; + + auto neighbors_stats = std::move(stats_.neighbors); + stats_.neighbors.clear(); + for (size_t i = 0; i < neighbors_stats.size(); ++i) { + if (!neighbors_.at(i).is_disabled()) { + stats_.neighbors.push_back(std::move(neighbors_stats[i])); + } + } +} + +/** + * This method is called when cell from shard state is loaded. + * Only if full_collated_data is set. + * When storage stat is calculated, StorageStatCalculationContext::calculating_storage_stat() returns true. + * In this case, loaded cell is stored in a separate StorageStat for the storage dict of the account (if it exists). + * + * @param loaded_cell Loaded cell + */ +void Collator::on_cell_loaded(const vm::LoadedCell& loaded_cell) { + auto context = block::StorageStatCalculationContext::get(); + vm::ProofStorageStat* stat = (context && context->calculating_storage_stat() && current_tx_storage_dict_ + ? ¤t_tx_storage_dict_->proof_stat + : &collated_data_stat); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); + } + stat->add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); + } +} + +/** + * Initializes current_tx_storage_dict_ to be used in on_cell_loaded. + * + * @param account Account of the current transaction + */ +void Collator::set_current_tx_storage_dict(const block::Account& account) { + if (!account.orig_storage_dict_hash) { + current_tx_storage_dict_ = nullptr; + return; + } + auto it = account_storage_dicts_.find(account.orig_storage_dict_hash.value()); + current_tx_storage_dict_ = it == account_storage_dicts_.end() ? nullptr : &it->second; +} + } // namespace validator } // namespace ton diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 8b1f5eb7f..54acb105e 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -112,6 +112,7 @@ void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); if (res.is_error()) { return res.move_as_error(); @@ -131,11 +131,11 @@ td::Result> create_ihr_message(td::BufferSlice data) { void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, int send_broadcast_mode, + td::Ref approve_signatures, int send_broadcast_mode, bool apply, td::actor::ActorId manager, td::Promise promise) { td::actor::create_actor( PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast_mode, manager, std::move(promise)) + std::move(signatures), std::move(approve_signatures), send_broadcast_mode, apply, manager, std::move(promise)) .release(); } @@ -192,57 +192,34 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) { +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } static std::atomic idx; - td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() - << ":" << (seqno + 1) << "#" << idx.fetch_add(1), - shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), std::move(manager), timeout, std::move(promise), - is_fake) + td::actor::create_actor( + PSTRING() << (params.is_fake ? "fakevalidate" : "validateblock") << params.shard.to_str() << ":" << (seqno + 1) + << "#" << idx.fetch_add(1), + std::move(candidate), std::move(params), std::move(manager), timeout, std::move(promise)) .release(); } -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) - << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), - shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), - creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - std::move(cancellation_token), mode, attempt_idx) - .release(); -} - -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { - BlockSeqno seqno = 0; - for (auto& p : prev) { - if (p.seqno() > seqno) { - seqno = p.seqno(); - } - } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, - min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) + td::actor::create_actor(PSTRING() << "collate" << params.shard.to_str() << ":" << (seqno + 1) + << (params.attempt_idx ? "_" + td::to_string(params.attempt_idx) : ""), + std::move(params), std::move(manager), timeout, std::move(cancellation_token), + std::move(promise)) .release(); } diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 4327b5dd3..75cf08bc7 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -99,7 +99,7 @@ td::Result> IhrMessageQ::create_ihr_message(td::BufferSlice dat "block header in the Merkle proof of an IHR message does not belong to the declared source block"); } vm::AugmentedDictionary out_msg_dict{vm::load_cell_slice_ref(extra.out_msg_descr), 256, - block::tlb::aug_OutMsgDescr}; + block::tlb::aug_OutMsgDescrDefault}; Bits256 key{ihr_msg->get_hash().bits()}; auto descr = out_msg_dict.lookup(key); out_msg_dict.reset(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 06f40b8fd..14e041464 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -917,7 +917,8 @@ void LiteQuery::continue_getLibraries(Ref mc_s mc_state_ = Ref(std::move(mc_state)); CHECK(mc_state_.not_null()); - auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), + block::ConfigInfo::needLibraries); if (rconfig.is_error()) { fatal_error("cannot extract library list block configuration from masterchain state"); return; @@ -1251,9 +1252,10 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, BlockIdExt& blkid, A return true; } -bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid) { - vm::MerkleProofBuilder mpb{std::move(state_root)}; - auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks); +bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid) { + vm::MerkleProofBuilder mpb{mc_state->root_cell()}; + auto rconfig = + block::ConfigInfo::extract_config(mpb.root(), mc_state->get_block_id(), block::ConfigInfo::needPrevBlocks); if (rconfig.is_error()) { return fatal_error( "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + @@ -1319,13 +1321,12 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); if (mode_ & 0x80000000) { - auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), 0xFFFF); if (config.is_error()) { fatal_error(config.move_as_error()); return; } auto rconfig = config.move_as_ok(); - rconfig->set_block_id_ext(mc_state_->get_block_id()); acc_state_promise_.set_value(std::make_tuple( std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) )); @@ -1432,13 +1433,16 @@ static td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, if (config && config->get_global_version() >= 6) { tuple.push_back(vm::StackEntry::maybe(config->get_unpacked_config_tuple(now))); // unpacked_config_tuple:[...] tuple.push_back(due_payment); // due_payment:Integer - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (my_code.not_null()) { precompiled = config->get_precompiled_contracts_config().get_contract(my_code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (config && config->get_global_version() >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); @@ -1499,7 +1503,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** auto r_config = block::ConfigInfo::extract_config( - mc_state_->root_cell(), + mc_state_->root_cell(), mc_state_->get_block_id(), block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); @@ -1902,7 +1906,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) if (mode & block::ConfigInfo::needPrevBlocks) { mode |= block::ConfigInfo::needCapabilities; } - auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + auto res = block::ConfigInfo::extract_config(mpb.root(), keyblk ? base_blk_id_ : mc_state_->get_block_id(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -2387,7 +2391,8 @@ void LiteQuery::perform_listBlockTransactions(BlockIdExt blkid, int mode, int co static td::Result> get_in_msg_metadata( const Ref& in_msg_descr_root, const Ref& trans_root) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { return td::Status::Error("invalid Transaction in block"); @@ -2546,7 +2551,8 @@ void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, const std::vector>& trans_roots) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; for (const Ref& trans_root : trans_roots) { block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { @@ -3042,7 +3048,7 @@ bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockId return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str()); } // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_` - if (!make_ancestor_block_proof(state_proof, mc_state_->root_cell(), next)) { + if (!make_ancestor_block_proof(state_proof, mc_state_, next)) { return fatal_error("cannot prove that "s + next.to_str() + " is in the previous block set of the masterchain state of " + cur.to_str()); } @@ -3672,7 +3678,7 @@ void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime fatal_error(r_messages_boc.move_as_error()); return; } - messages_boc = std::move(messages_boc); + messages_boc = r_messages_boc.move_as_ok(); } LOG(INFO) << "getDispatchQueueMessages(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; auto b = ton::create_serialize_tl_object( diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index fc8735332..afeabb9c0 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -221,7 +221,7 @@ class LiteQuery : public td::actor::Actor { bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); bool make_shard_info_proof(Ref& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix); - bool make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid); + bool make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid); }; } // namespace validator diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 95ad4a41e..dc56997a1 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -196,7 +196,7 @@ td::Result>> OutMsgQueueProof::fetch(Shard if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } - res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + res.emplace_back(true, blocks[i], state_root, block_state_proof, false, f.msg_counts_[i]); data[i].first = blocks[i]; TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); @@ -219,6 +219,342 @@ td::Result>> OutMsgQueueProof::fetch(Shard } } +void OutMsgQueueImporter::new_masterchain_block_notification(td::Ref state, + std::set collating_shards) { + last_masterchain_state_ = state; + if (collating_shards.empty() || state->get_unix_time() < (td::uint32)td::Clocks::system() - 20) { + return; + } + auto can_collate_shard = [&](const ShardIdFull& shard) -> bool { + return std::any_of(collating_shards.begin(), collating_shards.end(), + [&](ShardIdFull our_shard) { return shard_intersects(shard, our_shard); }); + }; + auto shards = state->get_shards(); + auto process_dst_shard = [&](const ShardIdFull& dst_shard) { + if (!can_collate_shard(dst_shard)) { + return; + } + std::vector top_blocks; + for (const auto& shard : shards) { + if (block::ShardConfig::is_neighbor(dst_shard, shard->shard())) { + top_blocks.push_back(shard->top_block_id()); + } + } + get_neighbor_msg_queue_proofs(dst_shard, std::move(top_blocks), td::Timestamp::in(15.0), + [](td::Result>>) {}); + }; + for (const auto& shard : shards) { + if (shard->before_merge()) { + if (is_left_child(shard->shard())) { + process_dst_shard(shard_parent(shard->shard())); + } + } else if (shard->before_split()) { + process_dst_shard(shard_child(shard->shard(), true)); + process_dst_shard(shard_child(shard->shard(), false)); + } else { + process_dst_shard(shard->shard()); + } + } +} + +void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + std::sort(blocks.begin(), blocks.end()); + auto entry = cache_[{dst_shard, blocks}]; + if (entry) { + if (entry->done) { + promise.set_result(entry->result); + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + } else { + entry->timeout = std::max(entry->timeout, timeout); + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(timeout); + } + return; + } + + FLOG(DEBUG) { + sb << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks:"; + for (const BlockIdExt& block : blocks) { + sb << " " << block.id.to_str(); + } + }; + + cache_[{dst_shard, blocks}] = entry = std::make_shared(); + entry->dst_shard = dst_shard; + entry->blocks = blocks; + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(entry->timeout = timeout); + + std::map> new_queries; + for (const BlockIdExt& block : blocks) { + if (opts_->need_monitor(block.shard_full(), last_masterchain_state_)) { + ++entry->pending; + get_proof_local(entry, block); + } else { + ShardIdFull prefix = block.shard_full(); + int min_split = last_masterchain_state_->monitor_min_split_depth(prefix.workchain); + if (prefix.pfx_len() > min_split) { + prefix = shard_prefix(prefix, min_split); + } + + LOG(DEBUG) << "search for out msg queue proof " << prefix.to_str() << " " << block.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, block)]; + if (!small_entry.result.is_null()) { + entry->result[block] = small_entry.result; + entry->from_small_cache++; + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + } else { + small_entry.pending_entries.push_back(entry); + ++entry->pending; + new_queries[prefix].push_back(block); + } + } + }; + auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); + for (auto& p : new_queries) { + for (size_t i = 0; i < p.second.size(); i += 16) { + size_t j = std::min(i + 16, p.second.size()); + get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), + limits * (td::uint32)(j - i)); + } + } + if (entry->pending == 0) { + finish_query(entry); + } +} + +void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, BlockIdExt block) { + if (!check_timeout(entry)) { + return; + } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, false, + [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, + retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block state for " << block.to_str() << ": " << R.move_as_error(); + delay_action([=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + auto state = R.move_as_ok(); + if (block.seqno() == 0) { + std::vector> proof = { + td::Ref(true, block, state->root_cell(), td::Ref{}, true)}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::wait_block_data_short, block, 0, timeout, + [=](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block data for " << block.to_str() << ": " << R.move_as_error(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); + std::vector> proof = { + td::Ref(true, block, state->root_cell(), std::move(block_state_proof), true)}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), + ProofSource::Local); + }); + }); +} + +void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits) { + if (!check_timeout(entry)) { + return; + } + td::actor::send_closure( + manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, entry->dst_shard, blocks, limits, + [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.1), + dst_shard = entry->dst_shard](td::Result>> R) { + if (R.is_error()) { + FLOG(DEBUG) { + sb << "Failed to get out msg queue for " << dst_shard.to_str() << " from"; + for (const BlockIdExt &block : blocks) { + sb << " " << block.id.to_str(); + } + sb << ": " << R.move_as_error(); + }; + delay_action( + [=]() { + td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_import, entry, std::move(blocks), + limits); + }, + retry_after); + return; + } + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok(), ProofSource::Query); + }); +} + +void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs, + ProofSource proof_source) { + if (!check_timeout(entry)) { + return; + } + // TODO: maybe save proof to small cache? It would allow other queries to reuse this result + for (auto& p : proofs) { + auto block_id = p->block_id_; + if (entry->result.emplace(block_id, std::move(p)).second) { + CHECK(entry->pending > 0); + switch (proof_source) { + case ProofSource::SmallCache: + entry->from_small_cache++; + break; + case ProofSource::Broadcast: + entry->from_broadcast++; + break; + case ProofSource::Query: + entry->from_query++; + break; + case ProofSource::Local: + entry->from_local++; + break; + } + if (--entry->pending == 0) { + finish_query(entry); + } + } + } +} + +void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { + FLOG(INFO) { + sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << " in " << entry->timer.elapsed() << "s"; + sb << " sources{"; + if (entry->from_broadcast) { + sb << " broadcast=" << entry->from_broadcast; + } + if (entry->from_small_cache) { + sb << " small_cache=" << entry->from_small_cache; + } + if (entry->from_local) { + sb << " local=" << entry->from_local; + } + if (entry->from_query) { + sb << " query=" << entry->from_query; + } + sb << "}"; + + if (!small_cache_.empty()) { + sb << " small_cache_size=" << small_cache_.size(); + } + if (!cache_.empty()) { + sb << " cache_size=" << cache_.size(); + } + }; + + entry->done = true; + CHECK(entry->blocks.size() == entry->result.size()); + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + for (auto& p : entry->promises) { + p.first.set_result(entry->result); + } + entry->promises.clear(); +} + +bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { + if (entry->timeout.is_in_past()) { + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; + for (auto& p : entry->promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + entry->promises.clear(); + auto it = cache_.find({entry->dst_shard, entry->blocks}); + if (it != cache_.end() && it->second == entry) { + cache_.erase(it); + } + return false; + } + return true; +} + +void OutMsgQueueImporter::alarm() { + for (auto it = cache_.begin(); it != cache_.end();) { + auto& promises = it->second->promises; + if (it->second->timeout.is_in_past()) { + if (!it->second->done) { + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : it->second->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; + for (auto& p : promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + promises.clear(); + } + it = cache_.erase(it); + continue; + } + alarm_timestamp().relax(it->second->timeout); + size_t j = 0; + for (auto& p : promises) { + if (p.second.is_in_past()) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } else { + alarm_timestamp().relax(p.second); + promises[j++] = std::move(p); + } + } + promises.resize(j); + ++it; + } + + for (auto it = small_cache_.begin(); it != small_cache_.end();) { + td::remove_if(it->second.pending_entries, + [](const std::shared_ptr& entry) { return entry->done || entry->promises.empty(); }); + if (it->second.timeout.is_in_past()) { + if (it->second.pending_entries.empty()) { + it = small_cache_.erase(it); + } else { + ++it; + } + } else { + alarm_timestamp().relax(it->second.timeout); + ++it; + } + } +} + +void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << " " << proof->block_id_.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, proof->block_id_)]; + if (!small_entry.result.is_null()) { + return; + } + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + small_entry.result = proof; + CHECK(proof.not_null()); + auto pending_entries = std::move(small_entry.pending_entries); + for (auto& entry : pending_entries) { + got_proof(entry, {proof}, ProofSource::Broadcast); + } +} + void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { FLOG(DEBUG) { @@ -235,6 +571,33 @@ void BuildOutMsgQueueProof::abort_query(td::Status reason) { } void BuildOutMsgQueueProof::start_up() { + if (blocks_.size() > 16) { + abort_query(td::Status::Error("too many blocks")); + return; + } + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_top_masterchain_state, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_masterchain_state, + R.move_as_ok()); + } + }); +} + +void BuildOutMsgQueueProof::got_masterchain_state(Ref mc_state) { + auto config_limits = mc_state->get_imported_msg_queue_limits(dst_shard_.is_masterchain()); + if ((td::uint64)config_limits.max_msgs * blocks_.size() < limits_.max_msgs) { + abort_query(td::Status::Error("too big max_msgs")); + return; + } + if ((td::uint64)config_limits.max_bytes * blocks_.size() < limits_.max_bytes) { + abort_query(td::Status::Error("too big max_bytes")); + return; + } + for (size_t i = 0; i < blocks_.size(); ++i) { BlockIdExt id = blocks_[i].id; ++pending; diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index e28561e2c..b537f31a9 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -31,6 +31,66 @@ using td::Ref; class ValidatorManager; class ValidatorManagerInterface; +class OutMsgQueueImporter : public td::actor::Actor { + public: + OutMsgQueueImporter(td::actor::ActorId manager, td::Ref opts, + td::Ref last_masterchain_state) + : manager_(manager), opts_(opts), last_masterchain_state_(last_masterchain_state) { + } + + void new_masterchain_block_notification(td::Ref state, std::set collating_shards); + void get_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise); + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + void alarm() override; + + private: + td::actor::ActorId manager_; + td::Ref opts_; + td::Ref last_masterchain_state_; + + struct CacheEntry { + ShardIdFull dst_shard; + std::vector blocks; + bool done = false; + std::map> result; + std::vector>>, td::Timestamp>> promises; + td::Timestamp timeout = td::Timestamp::never(); + td::Timer timer; + size_t pending = 0; + size_t from_small_cache = 0; + size_t from_broadcast = 0; + size_t from_query = 0; + size_t from_local = 0; + }; + std::map>, std::shared_ptr> cache_; + + // This cache has smaller granularity, proof is stored for each block separately + struct SmallCacheEntry { + td::Ref result; + std::vector> pending_entries; + td::Timestamp timeout = td::Timestamp::never(); + }; + std::map, SmallCacheEntry> small_cache_; + + void get_proof_local(std::shared_ptr entry, BlockIdExt block); + void get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits); + enum class ProofSource { + SmallCache, Broadcast, Query, Local + }; + void got_proof(std::shared_ptr entry, std::vector> proofs, ProofSource proof_source); + void finish_query(std::shared_ptr entry); + bool check_timeout(std::shared_ptr entry); + + constexpr static const double CACHE_TTL = 60.0; +}; + class BuildOutMsgQueueProof : public td::actor::Actor { public: BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, @@ -45,6 +105,7 @@ class BuildOutMsgQueueProof : public td::actor::Actor { void abort_query(td::Status reason); void start_up() override; + void got_masterchain_state(Ref mc_state); void got_state_root(size_t i, Ref root); void got_block_root(size_t i, Ref root); void build_proof(); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0ab216f71..5803eac89 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -376,9 +376,9 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( - root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | - block::ConfigInfo::needWorkchainInfo); + root_cell(), blkid, + block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | + block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -386,16 +386,12 @@ td::Status MasterchainStateQ::mc_reinit() { } config_ = res.move_as_ok(); CHECK(config_); - CHECK(config_->set_block_id_ext(get_block_id())); - auto cv_root = config_->get_config_param(35, 34); - if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); - cur_validators_ = std::move(validators); - } + cur_validators_ = config_->get_cur_validator_set(); + auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); next_validators_ = std::move(validators); } @@ -501,11 +497,11 @@ std::vector> MasterchainStateQ::get_shards() const { return v; } -td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard) const { +td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard, bool exact) const { if (!config_) { return {}; } - return config_->get_shard_hash(shard); + return config_->get_shard_hash(shard, exact); } bool MasterchainStateQ::rotated_all_shards() const { @@ -524,6 +520,14 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } +td::uint32 MasterchainStateQ::persistent_state_split_depth(WorkchainId workchain_id) const { + if (!config_) { + return 0; + } + auto wc_info = config_->get_workchain_info(workchain_id); + return wc_info.not_null() ? wc_info->persistent_state_split_depth : 0; +} + td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index fa36e1e66..7949de129 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -114,7 +114,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { Ref get_validator_set(ShardIdFull shard, UnixTime ts, CatchainSeqno cc_seqno) const; bool rotated_all_shards() const override; std::vector> get_shards() const override; - td::Ref get_shard_from_config(ShardIdFull shard) const override; + td::Ref get_shard_from_config(ShardIdFull shard, bool exact) const override; bool ancestor_is_valid(BlockIdExt id) const override { return check_old_mc_block_id(id); } @@ -124,6 +124,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const override; td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; @@ -138,6 +139,13 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { auto R = config_->get_size_limits_config(); return R.is_error() ? block::SizeLimitsConfig::ExtMsgLimits() : R.ok_ref().ext_msg_limits; } + block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const override { + auto R = config_->get_block_limits(is_masterchain); + if (R.is_ok() && R.ok()) { + return R.ok()->imported_msg_queue; + } + return {}; + } BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index 5c866a6e3..e1fd5778f 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -74,6 +74,9 @@ class ShardTopBlockDescrQ final : public ShardTopBlockDescrQBase { std::size_t size() const { return chain_blk_ids_.size(); } + const std::vector& get_chain_blocks() const override { + return chain_blk_ids_; + } UnixTime generated_at() const override { return gen_utime_; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 90966d820..a1ab5026f 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -31,6 +31,9 @@ #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" #include "common/errorlog.h" +#include "fabric.h" +#include "storage-stat-cache.hpp" + #include namespace ton { @@ -56,32 +59,29 @@ std::string ErrorCtx::as_string() const { /** * Constructs a ValidateQuery object. * - * @param shard The shard of the block being validated. - * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. - * @param prev A vector of BlockIdExt representing the previous blocks. * @param candidate The BlockCandidate to be validated. - * @param validator_set A reference to the ValidatorSet. + * @param params Validation parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param is_fake A boolean indicating if the validation is fake (performed when creating a hardfork). */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, Ref validator_set, +ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) - : shard_(shard) + td::Promise promise) + : shard_(params.shard) , id_(candidate.id) - , min_mc_block_id(min_masterchain_block_id) - , prev_blocks(std::move(prev)) + , min_mc_block_id(params.min_masterchain_block_id) + , prev_blocks(std::move(params.prev)) , block_candidate(std::move(candidate)) - , validator_set_(std::move(validator_set)) + , validator_set_(std::move(params.validator_set)) + , local_validator_id_(params.local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(is_fake) + , is_fake_(params.is_fake) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { @@ -115,7 +115,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(false); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -153,7 +153,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(false); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -176,7 +176,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { - record_stats(false); + record_stats(false, error.message().str()); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -346,33 +346,58 @@ void ValidateQuery::start_up() { // return; } } - // 2. load state(s) corresponding to previous block(s) - prev_states.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (optimistic_prev_block_.not_null()) { + if (is_masterchain()) { + fatal_error("optimistic validation in masterchain is not supported"); + return; + } + if (prev_blocks.size() != 1) { + fatal_error("optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error("optimistic prev block is not null, but has invalid block id"); + return; + } + LOG(WARNING) << "Optimistic prev block id = " << optimistic_prev_block_->block_id().to_str(); } + // 2. learn latest masterchain state and block id + LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); + }); // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { reject_query("error unpacking block candidate"); return; } + // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) + prev_states.resize(prev_blocks.size()); + if (is_masterchain() || !full_collated_data_) { + if (optimistic_prev_block_.is_null()) { + load_prev_states(); + } else { + if (!process_optimistic_prev_block()) { + return; + } + } + } // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, - [self = get_self()](td::Result res) { - LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_handle, mc_blkid_, true, + [self = get_self(), token = perf_log_.start_action("get_block_handle")](td::Result res) mutable { + LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, std::move(res), + std::move(token)); + }); } else { if (prev_blocks[0] != mc_blkid_) { soft_reject_query("cannot validate masterchain block "s + id_.to_str() + @@ -381,10 +406,106 @@ void ValidateQuery::start_up() { return; } } + // 5. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_storage_stat_cache, + std::move(res), std::move(token)); + }); // ... CHECK(pending); } +/** + * Load previous states from DB + */ +void ValidateQuery::load_prev_states() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 4.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i)]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + } +} + +/** + * Load previous state for optimistic prev block to apply Merkle update to it + */ +bool ValidateQuery::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 4.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state_merge")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); + } + return true; +} + +/** + * Callback function called after retrieving previous state for optimistic prev block + * + * @param res The retrieved state. + */ +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token) { + token.finish(res); + LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks and validates a block candidate. * @@ -606,6 +727,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (!ins.second) { return reject_query("Merkle proof with duplicate virtual root hash "s + virt_hash.to_hex()); } + full_collated_data_ = true; return true; } if (block::gen::t_TopBlockDescrSet.has_valid_tag(cs)) { @@ -619,6 +741,24 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } + if (block::gen::t_AccountStorageDictProof.has_valid_tag(cs)) { + if (!block::gen::t_AccountStorageDictProof.validate_upto(10000, cs)) { + return reject_query("invalid AccountStorageDictProof"); + } + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + Ref proof = cs.prefetch_ref(); + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return reject_query("invalid Merkle proof in AccountStorageDictProof"); + } + LOG(DEBUG) << "collated datum # " << idx << " is an AccountStorageDictProof with hash " + << virt_root->get_hash().to_hex(); + if (!virt_account_storage_dicts_.emplace(virt_root->get_hash().bits(), virt_root).second) { + return reject_query("duplicate AccountStorageDictProof"); + } + full_collated_data_ = true; + return true; + } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -643,6 +783,9 @@ bool ValidateQuery::extract_collated_data() { return reject_query(PSTRING() << "virtualization error " << err.get_msg()); } } + if (full_collated_data_) { + LOG(INFO) << "full_collated_data = true"; + } return true; } @@ -652,10 +795,12 @@ bool ValidateQuery::extract_collated_data() { void ValidateQuery::request_latest_mc_state() { ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); } @@ -664,7 +809,9 @@ void ValidateQuery::request_latest_mc_state() { * * @param res The result of the retrieval of the latest masterchain state. */ -void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res) { +void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_latest_mc_state()"; --pending; if (res.is_error()) { @@ -705,7 +852,8 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { +void ValidateQuery::after_get_mc_state(td::Result> res, td::PerfLogAction token) { + token.finish(res); CHECK(!is_masterchain()); LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; @@ -730,7 +878,8 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { * * @param res The result of retrieving the masterchain block handle. */ -void ValidateQuery::got_mc_handle(td::Result res) { +void ValidateQuery::got_mc_handle(td::Result res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); if (res.is_error()) { fatal_error(res.move_as_error()); @@ -738,24 +887,49 @@ void ValidateQuery::got_mc_handle(td::Result res) { } auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, - [self = get_self(), id = id_, mc_handle](td::Result> res) { + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, + [self = get_self(), id = id_, mc_handle, + token = perf_log_.start_action("mc wait_block_state")](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + id.to_str() + " does not have a valid proof"); } - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res), + std::move(token)); }); } +/** + * Callback function called after retrieving storage stat cache. + * + * @param res The retrieved storage stat cache. + */ +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { + token.finish(res); + --pending; + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + if (!pending) { + if (!try_validate()) { + fatal_error("cannot validate new block"); + } + } +} + /** * Callback function called after retrieving the shard state for a previous block. * * @param idx The index of the previous block (0 or 1). * @param res The result of the shard state retrieval. */ -void ValidateQuery::after_get_shard_state(int idx, td::Result> res) { +void ValidateQuery::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { @@ -835,7 +1009,7 @@ bool ValidateQuery::try_unpack_mc_state() { return fatal_error(-666, "latest masterchain state does not have a root cell"); } auto res = block::ConfigInfo::extract_config( - mc_state_root_, + mc_state_root_, mc_blkid_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -846,7 +1020,6 @@ bool ValidateQuery::try_unpack_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_blkid_); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { @@ -877,6 +1050,7 @@ bool ValidateQuery::try_unpack_mc_state() { return reject_query(PSTRING() << "vertical seqno mismatch: new block has " << vert_seqno_ << " while the masterchain configuration expects " << config_->get_vert_seqno()); } + global_version_ = config_->get_global_version(); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { prev_key_block_seqno_ = prev_key_block_.seqno(); @@ -977,6 +1151,7 @@ bool ValidateQuery::fetch_config_params() { compute_phase_cfg_.size_limits = size_limits; compute_phase_cfg_.precompiled_contracts = config_->get_precompiled_contracts_config(); compute_phase_cfg_.allow_external_unfreeze = compute_phase_cfg_.global_version >= 8; + compute_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; } { // compute action_phase_cfg @@ -1005,9 +1180,15 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; + action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; + action_phase_cfg_.global_version = config_->get_global_version(); } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; + serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; + serialize_cfg_.size_limits = size_limits; } { // fetch block_grams_created @@ -1231,6 +1412,20 @@ bool ValidateQuery::check_this_shard_mc_info() { */ bool ValidateQuery::compute_prev_state() { CHECK(prev_states.size() == 1u + after_merge_); + CHECK(prev_states.size() == prev_blocks.size()); + if (!is_masterchain() && full_collated_data_) { + for (size_t i = 0; i < prev_states.size(); i++) { + Ref root = get_virt_state_root(prev_blocks[i]); + if (root.is_null()) { + return reject_query(PSTRING() << "cannot get previous state from collated data: " << prev_blocks[i].to_str()); + } + auto r_state = create_shard_state(prev_blocks[i], std::move(root)); + if (r_state.is_error()) { + return reject_query("failed to parse previous state from collated data", r_state.move_as_error()); + } + prev_states[i] = r_state.move_as_ok(); + } + } // Extend validator timeout if previous block is too old UnixTime prev_ts = prev_states[0]->get_unix_time(); if (after_merge_) { @@ -1321,17 +1516,17 @@ bool ValidateQuery::compute_next_state() { } } auto r_config_info = block::ConfigInfo::extract_config( - state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | - block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | - block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needAccountsRoot | - block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities); + state_root_, id_, + block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | + block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | + block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc | + block::ConfigInfo::needCapabilities); if (r_config_info.is_error()) { return reject_query("cannot extract configuration from new masterchain state "s + mc_blkid_.to_str() + " : " + r_config_info.error().to_string()); } new_config_ = r_config_info.move_as_ok(); CHECK(new_config_); - new_config_->set_block_id_ext(id_); } return true; } @@ -1492,6 +1687,9 @@ bool ValidateQuery::request_neighbor_queues() { auto neighbor_list = new_shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = new_shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return reject_query("cannot obtain shard hash for neighbor "s + blk_id.to_str()); @@ -1503,15 +1701,50 @@ bool ValidateQuery::request_neighbor_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; - for (block::McShardDescr& descr : neighbors_) { - LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); - ++i; + if (full_collated_data_) { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); + if (descr.blk_.is_masterchain()) { + if (descr.blk_ != mc_state_->get_block_id()) { + return fatal_error("neighbor from masterchain is not the last mc block"); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue(), + td::PerfLogAction{}); + ++i; + continue; + } + auto state_root = get_virt_state_root(descr.blk_); + if (state_root.is_null()) { + return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_.to_str()); + } + auto state = ShardStateQ::fetch(descr.blk_, {}, std::move(state_root)); + if (state.is_error()) { + return reject_query("cannot fetch shard state from collated data", state.move_as_error()); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue(), + td::PerfLogAction{}); + ++i; + } + } else { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); + ++pending; + if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { + td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, + prev_states.at(prev_idx)->message_queue(), td::PerfLogAction{}); + } else { + send_closure_later( + manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_message_queue #" << i)]( + td::Result> res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, std::move(res), + std::move(token)); + }); + } + ++i; + } } return true; } @@ -1523,7 +1756,8 @@ bool ValidateQuery::request_neighbor_queues() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res) { +void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -1650,13 +1884,15 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_aux_shard_state, blkid, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, + [self = get_self(), blkid, + token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno())]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -1686,7 +1922,9 @@ Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; if (res.is_error()) { @@ -1720,11 +1958,13 @@ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result< * @param sibling The sibling shard information. * @param wc_info The workchain information. * @param ccvc The Catchain validators configuration. + * @param is_new Set to true if the top shard block is new, false if it existed in the previous shard configuration. * * @returns True if the validation wasa successful, false otherwise. */ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc) { + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, + bool& is_new) { auto shard = info.shard(); LOG(DEBUG) << "checking shard " << shard.to_str() << " in new shard configuration"; if (info.next_validator_shard_ != shard.shard) { @@ -1733,6 +1973,7 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block: ton::ShardIdFull{shard.workchain, info.next_validator_shard_}.to_str()); } auto old = old_shard_conf_->get_shard_hash(shard - 1, false); + is_new = (old.is_null() || old->top_block_id() != info.top_block_id()); Ref prev; CatchainSeqno cc_seqno; bool old_before_merge = false, fsm_inherited = false, workchain_created = false; @@ -2032,8 +2273,10 @@ bool ValidateQuery::check_shard_layout() { WorkchainId wc_id{ton::workchainInvalid}; Ref wc_info; - if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &wc_set, &wc_id, &wc_info, &ccvc]( - block::McShardHash& cur, const block::McShardHash* sibling) { + std::vector new_top_shard_blocks; + if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &new_top_shard_blocks, &wc_set, &wc_id, &wc_info, + &ccvc](block::McShardHash& cur, + const block::McShardHash* sibling) { if (!cur.is_valid()) { return -2; } @@ -2050,7 +2293,15 @@ bool ValidateQuery::check_shard_layout() { wc_info = it->second; } } - return self->check_one_shard(cur, sibling, wc_info.get(), ccvc) ? 0 : -1; + bool is_new; + int res = self->check_one_shard(cur, sibling, wc_info.get(), ccvc, is_new); + if (!res) { + return -1; + } + if (is_new) { + new_top_shard_blocks.push_back(cur.top_block_id()); + } + return 0; })) { return reject_query("new shard configuration is invalid"); } @@ -2067,6 +2318,15 @@ bool ValidateQuery::check_shard_layout() { << " is active, but is absent from new shard configuration"); } } + if (!new_top_shard_blocks.empty()) { + ++pending; + td::actor::send_closure( + manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this), + token = perf_log_.start_action("wait_verify_shard_blocks")](td::Result R) mutable { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, R.move_as_status(), std::move(token)); + }); + } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2223,16 +2483,30 @@ bool ValidateQuery::prepare_out_msg_queue_size() { if (ps_.out_msg_queue_size_) { // if after_split then out_msg_queue_size is always present, since it is calculated during split old_out_msg_queue_size_ = ps_.out_msg_queue_size_.value(); + out_msg_queue_size_known_ = true; + have_out_msg_queue_size_in_state_ = true; + return true; + } + if (ps_.out_msg_queue_->is_empty()) { + old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; + have_out_msg_queue_size_in_state_ = true; + return true; + } + if (!store_out_msg_queue_size_) { // Don't need it return true; } old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; - send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], - [self = get_self(), i](td::Result res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "get_out_msg_queue_size #" << i)]( + td::Result res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, std::move(res), + std::move(token)); + }); } return true; } @@ -2245,7 +2519,8 @@ bool ValidateQuery::prepare_out_msg_queue_size() { * @param i The index of the previous block (0 or 1). * @param res The result object containing the size of the queue. */ -void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { +void ValidateQuery::got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error( @@ -2258,6 +2533,24 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { try_validate(); } +/** + * Handles the result of ValidatorManager::wait_verify_shard_blocks. + * + * This is called after new top shard blocks were confirmed by trusted nodes. + * + * @param S The status of the operation (OK on success). + */ +void ValidateQuery::verified_shard_blocks(td::Status S, td::PerfLogAction token) { + token.finish(S); + --pending; + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); + return; + } + LOG(DEBUG) << "Verified shard blocks"; + try_validate(); +} + /* * * METHODS CALLED FROM try_validate() stage 1 @@ -2538,17 +2831,19 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { + t_InMsgDescr.aug.global_version = global_version_; + t_OutMsgDescr.aug.global_version = global_version_; + if (!t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { + if (!t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } - in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); - out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, block::tlb::aug_OutMsgDescr); + in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, t_InMsgDescr.aug); + out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, t_OutMsgDescr.aug); account_blocks_dict_ = std::make_unique( vm::load_cell_slice_ref(std::move(extra.account_blocks)), 256, block::tlb::aug_ShardAccountBlocks); LOG(DEBUG) << "validating InMsgDescr"; @@ -2834,7 +3129,7 @@ bool ValidateQuery::precheck_account_updates() { CHECK(key_len == 256); return precheck_one_account_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid ShardAccounts dictionary in the new state"); } } catch (vm::VmError& err) { @@ -3131,7 +3426,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } ton::LogicalTime enqueued_lt = old_value->prefetch_ulong(64); if (enqueued_lt >= start_lt_) { - return reject_query(PSTRING() << "new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" + return reject_query(PSTRING() << "old EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" << enqueued_lt << " greater than or equal to this block's start_lt=" << start_lt_); } } @@ -3281,14 +3576,16 @@ bool ValidateQuery::precheck_message_queue_update() { CHECK(key_len == 352); return precheck_one_message_queue_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid OutMsgQueue dictionary in the new state"); } } catch (vm::VmError& err) { return reject_query("invalid OutMsgQueue dictionary difference between the old and the new state: "s + err.get_msg()); } - LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + if (out_msg_queue_size_known_) { + LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + } if (store_out_msg_queue_size_) { if (!ns_.out_msg_queue_size_) { return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " @@ -3440,12 +3737,13 @@ bool ValidateQuery::unpack_dispatch_queue_update() { return check_account_dispatch_queue_update(key, ps_.dispatch_queue_->extract_value(std::move(old_val_extra)), ns_.dispatch_queue_->extract_value(std::move(new_val_extra))); }, - 3 /* check augmentation of changed nodes */); + 2 /* check augmentation of changed nodes in the new dict */); if (!res) { return reject_query("invalid DispatchQueue dictionary in the new state"); } - if (old_out_msg_queue_size_ <= compute_phase_cfg_.size_limits.defer_out_queue_size_limit) { + if (have_out_msg_queue_size_in_state_ && + old_out_msg_queue_size_ <= compute_phase_cfg_.size_limits.defer_out_queue_size_limit) { // Check that at least one message was taken from each AccountDispatchQueue try { have_unprocessed_account_dispatch_queue_ = false; @@ -3604,7 +3902,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -4166,7 +4464,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -4806,6 +5104,7 @@ bool ValidateQuery::check_processed_upto() { if (!ok) { return reject_query("new ProcessedInfo is not obtained from old ProcessedInfo by adding at most one new entry"); } + processed_upto_updated_ = upd; if (upd) { if (upd->shard != shard_.shard) { return reject_query("newly-added ProcessedInfo entry refers to shard "s + @@ -4882,16 +5181,8 @@ bool ValidateQuery::check_dispatch_queue_update() { */ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& nb, - bool& unprocessed) { + bool& unprocessed, bool& processed_here, td::Bits256& msg_hash) { unprocessed = false; - if (!block::gen::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass automated validity tests"); - } - if (!block::tlb::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass hand-written validity tests"); - } block::EnqueuedMsgDescr enq; if (!enq.unpack(enq_msg.write())) { // unpack EnqueuedMsg return reject_query("cannot unpack EnqueuedMsg with key "s + key.to_hex(352) + @@ -4911,6 +5202,8 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, auto out_entry = out_msg_dict_->lookup(key + 96, 256); bool f0 = ps_.processed_upto_->already_processed(enq); bool f1 = ns_.processed_upto_->already_processed(enq); + processed_here = f1 && !f0; + msg_hash = enq.hash_; if (f0 && !f1) { return fatal_error( "a previously processed message has been un-processed (impossible situation after the validation of " @@ -5024,7 +5317,26 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, * @returns True if the messages are valid, false otherwise. */ bool ValidateQuery::check_in_queue() { - block::OutputQueueMerger nb_out_msgs(shard_, neighbors_); + int imported_messages_count = 0; + in_msg_dict_->check_for_each_extra([&](Ref value, Ref, td::ConstBitPtr, int) { + int tag = block::gen::t_InMsg.get_tag(*value); + if (tag == block::gen::InMsg::msg_import_fin || tag == block::gen::InMsg::msg_import_tr) { + ++imported_messages_count; + } + return true; + }); + if (imported_messages_count == 0 && claimed_proc_lt_ == 0) { + return true; + } + + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + td::BitArray<96> key; + key.bits().store_int(descr.workchain(), 32); + (key.bits() + 32).store_uint(descr.shard().shard, 64); + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); + } + block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); @@ -5037,7 +5349,10 @@ bool ValidateQuery::check_in_queue() { }; } bool unprocessed = false; - if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { + bool processed_here = false; + td::Bits256 msg_hash; + if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed, + processed_here, msg_hash)) { if (verbosity > 1) { FLOG(INFO) { sb << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source @@ -5048,6 +5363,13 @@ bool ValidateQuery::check_in_queue() { return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); } + if (processed_here) { + --imported_messages_count; + } + auto msg_lt = kv->lt; + if (imported_messages_count == 0 && msg_lt == claimed_proc_lt_ && msg_hash == claimed_proc_hash_) { + return true; + } if (unprocessed) { return true; } @@ -5153,9 +5475,61 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad << " does not really belong to current shard"); return {}; } + if (new_acc->storage_dict_hash) { + if (full_collated_data_ && !is_masterchain()) { + auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != virt_account_storage_dicts_.end()) { + LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) + << ", hash=" << it->second->get_hash().to_hex(); + auto S = new_acc->init_account_storage_stat(it->second); + if (S.is_error()) { + reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), + std::move(S)); + return {}; + } + } + } else if (storage_stat_cache_ && new_acc->storage_dict_hash) { + auto dict_root = storage_stat_cache_(new_acc->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = new_acc->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return {}; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" + << new_acc->storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += new_acc->storage_used.cells; + } else if (new_acc->storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += new_acc->storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += new_acc->storage_used.cells; + } + } + } return new_acc; } +/** + * Gets IHR fee of the internal message + * + * In earlier versions (before 12) the field extra_flags was ihr_fee. + * Since version 12 ihr_fee is always zero. + * + * @param info CommonMsgInfo of the internal message + * @param global_version global version from ConfigParam 8 + * + * @returns IHR fee + */ +static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : block::tlb::t_Grams.as_integer(std::move(info.extra_flags)); +} + /** * Checks the validity of a single transaction for a given account. * Performs transaction execution. @@ -5243,7 +5617,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT CHECK(money_imported.validate_unpack(info.value)); ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { - money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); + money_imported += get_ihr_fee(info, global_version_); } CHECK(money_imported.is_valid()); } @@ -5312,7 +5686,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); - msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); + msg_export_value += get_ihr_fee(info, global_version_); msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; @@ -5561,6 +5935,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // .... std::unique_ptr trs = std::make_unique(account, trans_type, lt, now_, in_msg_root); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trs->time_tvm; + stats_.work_time.trx_storage_stat += trs->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; + }; if (in_msg_root.not_null()) { if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { // inbound external message was not accepted @@ -5741,6 +6121,12 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re })) { return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); } + if ((!full_collated_data_ || is_masterchain()) && account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), + account.storage_used.cells); + } if (is_masterchain() && account.libraries_changed()) { return scan_account_libraries(account.orig_library, account.library, acc_addr); } else { @@ -5918,9 +6304,12 @@ bool ValidateQuery::check_special_message(Ref in_msg_root, const block if (block::tlb::t_Grams.as_integer(info.fwd_fee)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero fwd_fee"); } - if (block::tlb::t_Grams.as_integer(info.ihr_fee)->sgn()) { + if (get_ihr_fee(info, global_version_)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero ihr_fee"); } + if (block::tlb::t_Grams.as_integer(info.extra_flags)->sgn()) { + return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero extra_flags"); + } block::CurrencyCollection value; if (!value.validate_unpack(info.value)) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has an invalid value"); @@ -6775,6 +7164,25 @@ bool ValidateQuery::postcheck_value_flow() { return true; } +Ref ValidateQuery::get_virt_state_root(const BlockIdExt& block_id) { + auto it = virt_roots_.find(block_id.root_hash); + if (it == virt_roots_.end()) { + return {}; + } + Ref root = it->second; + if (block_id.seqno() == 0) { + return root; + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(root, block)) { + return {}; + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + td::Bits256 state_root_hash = upd_cs.prefetch_ref(1)->get_hash(0).bits(); + it = virt_roots_.find(state_root_hash); + return it == virt_roots_.end() ? Ref{} : it->second; +} + /** * MAIN VALIDATOR FUNCTION (invokes other methods in a suitable order). * @@ -6785,10 +7193,8 @@ bool ValidateQuery::try_validate() { return true; } work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; try { if (!stage_) { @@ -6897,7 +7303,7 @@ bool ValidateQuery::try_validate() { } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { - return fatal_error(-666, err.get_msg()); + return reject_query(err.get_msg()); } return save_candidate(); } @@ -6908,16 +7314,21 @@ bool ValidateQuery::try_validate() { * @returns True. */ bool ValidateQuery::save_candidate() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ValidateQuery::written_candidate); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), token = perf_log_.start_action("set_block_candidate")](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ValidateQuery::written_candidate, std::move(token)); + } + }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } @@ -6925,20 +7336,35 @@ bool ValidateQuery::save_candidate() { * Callback function called after saving block candidate. * Finishes validation. */ -void ValidateQuery::written_candidate() { +void ValidateQuery::written_candidate(td::PerfLogAction token) { + token.finish(td::Status::OK()); finish_query(); } /** * Sends validation work time to manager. */ -void ValidateQuery::record_stats(bool success) { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); +void ValidateQuery::record_stats(bool valid, std::string error_message) { + stats_.block_id = id_; + stats_.collated_data_hash = block_candidate.collated_file_hash; + stats_.validated_at = td::Clocks::system(); + stats_.self = local_validator_id_; + stats_.valid = valid; + if (valid) { + stats_.comment = (PSTRING() << "OK ts=" << now_); + } else { + stats_.comment = std::move(error_message); + } + stats_.actual_bytes = (td::uint32)block_candidate.data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); + stats_.total_time = perf_timer_.elapsed(); + stats_.work_time.total = work_timer_.elapsed_both(); + stats_.time_stats = (PSTRING() << perf_log_); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, - cpu_work_time, success); + LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real + << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + LOG(WARNING) << perf_log_; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 60f0cc8a4..f338f975c 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,9 @@ */ #pragma once + +#include "block-parse.h" +#include "fabric.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -112,15 +115,13 @@ class ValidateQuery : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + return capCreateStatsEnabled | capBounceMsgBody | capReportVersion | capShortDequeue | capStoreOutMsgQueueSize | + capMsgMetadata | capDeferMessages | capFullCollatedData; } public: - ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); + ValidateQuery(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); private: int verbosity{3 * 1}; @@ -132,6 +133,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> prev_states; BlockCandidate block_candidate; td::Ref validator_set_; + PublicKeyHash local_validator_id_ = PublicKeyHash::zero(); td::actor::ActorId manager; td::Timestamp timeout; td::Promise main_promise; @@ -143,6 +145,7 @@ class ValidateQuery : public td::actor::Actor { bool is_key_block_{false}; bool update_shard_cc_{false}; bool is_fake_{false}; + bool full_collated_data_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; bool outq_cleanup_partial_{false}; @@ -151,6 +154,7 @@ class ValidateQuery : public td::actor::Actor { td::BitArray<64> shard_pfx_; int shard_pfx_len_; td::Bits256 created_by_; + Ref optimistic_prev_block_; Ref prev_state_root_; Ref state_root_; @@ -168,6 +172,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> collated_roots_; std::map> virt_roots_; std::unique_ptr top_shard_descr_dict_; + std::map> virt_account_storage_dicts_; Ref shard_hashes_; // from McBlockExtra Ref blk_config_params_; // from McBlockExtra @@ -187,6 +192,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime max_shard_lt_{0}; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno vert_seqno_{~0U}; bool ihr_enabled_{false}; bool create_stats_enabled_{false}; @@ -212,12 +218,15 @@ class ValidateQuery : public td::actor::Actor { std::map> aux_mc_states_; block::ShardState ps_, ns_; + bool processed_upto_updated_{false}; std::unique_ptr sibling_out_msg_queue_; std::shared_ptr sibling_processed_upto_; std::map block_create_count_; unsigned block_create_total_{0}; + block::tlb::InMsgDescr t_InMsgDescr{0}; + block::tlb::OutMsgDescr t_OutMsgDescr{0}; std::unique_ptr in_msg_dict_, out_msg_dict_, account_blocks_dict_; block::ValueFlow value_flow_; block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0}; @@ -235,6 +244,11 @@ class ValidateQuery : public td::actor::Actor { std::map, Ref> new_dispatch_queue_messages_; std::set account_expected_defer_all_messages_; td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + bool out_msg_queue_size_known_ = false; + bool have_out_msg_queue_size_in_state_ = false; + + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; bool msg_metadata_enabled_ = false; bool deferring_messages_enabled_ = false; @@ -244,6 +258,7 @@ class ValidateQuery : public td::actor::Actor { bool have_unprocessed_account_dispatch_queue_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; static constexpr td::uint32 priority() { return 2; @@ -260,8 +275,12 @@ class ValidateQuery : public td::actor::Actor { void alarm() override; void start_up() override; + void load_prev_states(); + bool process_optimistic_prev_block(); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); + bool save_candidate(); - void written_candidate(); + void written_candidate(td::PerfLogAction token); bool fatal_error(td::Status error); bool fatal_error(int err_code, std::string err_msg); @@ -281,15 +300,25 @@ class ValidateQuery : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } td::actor::ActorId get_self() { return actor_id(this); } void request_latest_mc_state(); - void after_get_latest_mc_state(td::Result, BlockIdExt>> res); - void after_get_mc_state(td::Result> res); - void got_mc_handle(td::Result res); - void after_get_shard_state(int idx, td::Result> res); + void after_get_latest_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_mc_state(td::Result> res, td::PerfLogAction token); + void got_mc_handle(td::Result res, td::PerfLogAction token); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); bool fetch_config_params(); @@ -309,22 +338,23 @@ class ValidateQuery : public td::actor::Actor { bool unpack_one_prev_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root); bool split_prev_state(block::ShardState& ss); bool request_neighbor_queues(); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc); + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); bool check_shard_layout(); bool register_shard_block_creators(std::vector creator_list); bool check_cur_validator_set(); bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); bool prepare_out_msg_queue_size(); - void got_out_queue_size(size_t i, td::Result res); + void got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token); + void verified_shard_blocks(td::Status S, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); @@ -360,7 +390,8 @@ class ValidateQuery : public td::actor::Actor { bool check_dispatch_queue_update(); bool check_processed_upto(); bool check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb, bool& unprocessed); + const block::McShardDescr& src_nb, bool& unprocessed, bool& processed_here, + td::Bits256& msg_hash); bool check_in_queue(); bool check_delivered_dequeued(); std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account); @@ -391,6 +422,8 @@ class ValidateQuery : public td::actor::Actor { const block::CurrencyCollection& create); bool check_mc_block_extra(); + Ref get_virt_state_root(const BlockIdExt& block_id); + bool check_timeout() { if (timeout && timeout.is_in_past()) { abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -399,9 +432,9 @@ class ValidateQuery : public td::actor::Actor { return true; } - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; - void record_stats(bool success); + td::RealCpuTimer work_timer_{true}; + ValidationStats stats_; + void record_stats(bool valid, std::string error_message = ""); }; } // namespace validator diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 951ca4b71..81c0bd518 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -74,7 +74,7 @@ class ValidatorSetCompute { private: const block::Config* config_{nullptr}; - std::unique_ptr cur_validators_, next_validators_; + std::shared_ptr cur_validators_, next_validators_; td::Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno seqno) const; }; diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp new file mode 100644 index 000000000..945f41a6b --- /dev/null +++ b/validator/import-db-slice-local.cpp @@ -0,0 +1,640 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "import-db-slice-local.hpp" + +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" +#include "ton/ton-io.hpp" +#include "downloaders/download-state.hpp" +#include "block/block-auto.h" + +#include + +namespace ton { + +namespace validator { + +ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Ref state, + BlockSeqno shard_client_seqno, td::Ref opts, + td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , promise_(std::move(promise)) + , perf_timer_("import-slice-local", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice-local", duration); + }) { +} + +void ArchiveImporterLocal::start_up() { + LOG(WARNING) << "Importing archive for masterchain seqno #" << shard_client_seqno_ + 1 << " from disk"; + for (const std::string &path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path); + if (S.is_error()) { + LOG(WARNING) << "Error processing package " << path << ": " << S; + } + } + + process_masterchain_blocks(); +} + +td::Status ArchiveImporterLocal::process_package(std::string path) { + LOG(DEBUG) << "Processing package " << path; + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + S = F.move_as_error(); + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt block_id; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + block_id = p.block_id; + ignore = !block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + block_id = p.block_id; + ignore = block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + block_id = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &) { ignore = true; })); + + if (ignore || block_id.is_masterchain() && block_id.seqno() <= last_masterchain_state_->get_seqno()) { + return true; + } + + if (is_proof) { + if (block_id.is_masterchain()) { + auto R = create_proof(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof = R.move_as_ok(); + } else { + auto R = create_proof_link(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof_link = R.move_as_ok(); + } + } else { + if (td::sha256_bits256(data) != block_id.file_hash) { + S = td::Status::Error(ErrorCode::protoviolation, "bad block file hash"); + return false; + } + auto R = create_block(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].block = R.move_as_ok(); + } + if (block_id.is_masterchain()) { + masterchain_blocks_[block_id.seqno()] = block_id; + } + return true; + }); + return S; +} + +void ArchiveImporterLocal::process_masterchain_blocks() { + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + + if (masterchain_blocks_.begin()->first != last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, PSTRING() << "expected masterchain seqno " + << last_masterchain_state_->get_seqno() + 1 + << ", found " << masterchain_blocks_.begin()->first)); + return; + } + { + BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; + for (auto &[seqno, _] : masterchain_blocks_) { + if (seqno != expected_seqno) { + abort_query( + td::Status::Error(ErrorCode::protoviolation, "non-consecutive masterchain blocks in the archive")); + return; + } + ++expected_seqno; + } + } + BlockInfo &first_block = blocks_[masterchain_blocks_.begin()->second]; + if (first_block.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (first_block.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + block::gen::Block::Record rec; + block::gen::BlockInfo::Record info; + if (!(block::gen::unpack_cell(first_block.block->root_cell(), rec) && block::gen::unpack_cell(rec.info, info))) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "cannot unpack masterchain block info")); + return; + } + if (info.key_block) { + import_first_key_block(); + return; + } + + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::import_first_key_block() { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + BlockInfo &first_block = blocks_[block_id]; + LOG(INFO) << "First block in archive is key block : " << block_id.id.to_str(); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), prev_block_id = last_masterchain_state_->get_block_id()]( + td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); + }); + + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(600.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); +} + +void ArchiveImporterLocal::checked_key_block_proof(BlockHandle handle) { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + CHECK(block_id == handle->id()); + BlockInfo &first_block = blocks_[block_id]; + run_apply_block_query( + handle->id(), first_block.block, handle->id(), manager_, td::Timestamp::in(600.0), + [SelfId = actor_id(this), manager = manager_, handle](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db, handle, [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_key_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_key_block(td::Ref state) { + CHECK(state->get_block_id() == masterchain_blocks_.begin()->second); + last_masterchain_state_ = state; + imported_any_ = true; + masterchain_blocks_.erase(masterchain_blocks_.begin()); + blocks_.erase(state->get_block_id()); + LOG(INFO) << "Imported key block " << state->get_block_id().id.to_str(); + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No more masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::process_masterchain_blocks_cont() { + LOG(INFO) << "Importing masterchain blocks from " << masterchain_blocks_.begin()->first << " to " + << masterchain_blocks_.rbegin()->first; + + td::MultiPromise mp; + auto ig = mp.init_guard(); + + BlockIdExt prev_block_id = last_masterchain_state_->get_block_id(); + for (auto &[_, block_id] : masterchain_blocks_) { + auto &info = blocks_[block_id]; + info.import = true; + if (info.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (info.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), prev_block_id, promise = ig.get_promise()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + promise.set_result(td::Unit()); + }); + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(600.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); + prev_block_id = block_id; + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + LOG(INFO) << "Checked proofs for masterchain blocks"; + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_masterchain_proofs); + } + }); +} + +void ArchiveImporterLocal::checked_masterchain_proofs() { + if (shard_client_seqno_ == last_masterchain_state_->get_seqno()) { + got_shard_client_state(last_masterchain_state_); + } else { + CHECK(shard_client_seqno_ < last_masterchain_state_->get_seqno()); + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(shard_client_seqno_, block_id)) { + abort_query(td::Status::Error("failed to get shard client block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get shard client state: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::got_shard_client_state, + td::Ref{R.move_as_ok()}); + }); + } +} + +void ArchiveImporterLocal::got_shard_client_state(td::Ref state) { + CHECK(state->get_seqno() == shard_client_seqno_); + LOG(DEBUG) << "got_shard_client_state " << shard_client_seqno_; + shard_client_state_ = state; + new_shard_client_seqno_ = shard_client_seqno_; + current_shard_client_seqno_ = shard_client_seqno_; + for (auto &shard : state->get_shards()) { + visited_shard_blocks_.insert(shard->top_block_id()); + } + try_advance_shard_client_seqno(); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno() { + BlockSeqno seqno = new_shard_client_seqno_ + 1; + auto it = masterchain_blocks_.find(seqno); + if (it != masterchain_blocks_.end()) { + try_advance_shard_client_seqno_cont(blocks_[it->second].block); + return; + } + if (seqno > last_masterchain_state_->get_seqno()) { + processed_shard_blocks(); + return; + } + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(seqno, block_id)) { + abort_query(td::Status::Error("failed to get old mc block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::try_advance_shard_client_seqno_cont, + R.move_as_ok()); + }); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno_cont(td::Ref mc_block) { + CHECK(mc_block.not_null()); + CHECK(mc_block->block_id().seqno() == new_shard_client_seqno_ + 1); + LOG(DEBUG) << "try_advance_shard_client_seqno " << new_shard_client_seqno_ + 1; + + block::gen::Block::Record rec; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + CHECK(block::gen::unpack_cell(mc_block->root_cell(), rec) && block::gen::unpack_cell(rec.extra, extra) && + block::gen::unpack_cell(extra.custom->prefetch_ref(), mc_extra)); + auto shard_config = std::make_unique(mc_extra.shard_hashes->prefetch_ref()); + + std::vector blocks_to_import; + std::function dfs = [&](const BlockIdExt &block_id) -> td::Status { + if (visited_shard_blocks_.contains(block_id)) { + return td::Status::OK(); + } + if (block_id.seqno() == 0) { + new_zerostates_.insert(block_id); + return td::Status::OK(); + } + visited_shard_blocks_.insert(block_id); + auto &info = blocks_[block_id]; + if (info.block.is_null()) { + return td::Status::Error(PSTRING() << "no shard block " << block_id.to_str()); + } + blocks_to_import.push_back(block_id); + + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS(block::unpack_block_prev_blk_try(info.block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt &prev_block_id : prev) { + TRY_STATUS(dfs(prev_block_id)); + } + + return td::Status::OK(); + }; + td::Status S = td::Status::OK(); + std::vector top_shard_blocks; + shard_config->process_shard_hashes([&](block::McShardHash &shard) { + if (!opts_->need_monitor(shard.shard(), shard_client_state_)) { + return 0; + } + S = dfs(shard.top_block_id()); + top_shard_blocks.push_back(shard.top_block_id()); + if (S.is_error()) { + return -1; + } + return 0; + }); + if (S.is_error()) { + LOG(DEBUG) << "Cannot advance shard client seqno to " << new_shard_client_seqno_ + 1 << " : " << S; + processed_shard_blocks(); + return; + } + shard_configs_[mc_block->block_id().seqno()] = {mc_block->block_id(), std::move(top_shard_blocks)}; + ++new_shard_client_seqno_; + LOG(DEBUG) << "Advancing shard client seqno to " << new_shard_client_seqno_; + for (const BlockIdExt &block_id : blocks_to_import) { + blocks_[block_id].import = true; + } + td::actor::send_closure(actor_id(this), &ArchiveImporterLocal::try_advance_shard_client_seqno); +} + +void ArchiveImporterLocal::processed_shard_blocks() { + if (new_shard_client_seqno_ == shard_client_seqno_) { + LOG(INFO) << "No new shard blocks"; + } else { + LOG(INFO) << "New shard client seqno = " << new_shard_client_seqno_; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (const BlockIdExt &block_id : new_zerostates_) { + LOG(INFO) << "Downloading zerostate " << block_id.to_str(); + td::actor::create_actor( + "downloadstate", block_id, shard_client_state_->get_block_id(), + shard_client_state_->persistent_state_split_depth(block_id.id.workchain), 2, manager_, td::Timestamp::in(3600), + ig.get_promise().wrap([](td::Ref &&) { return td::Unit(); })) + .release(); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::store_data); + } + }); +} + +void ArchiveImporterLocal::store_data() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + if (opts_->get_permanent_celldb()) { + std::vector> blocks; + for (auto &[_, info] : blocks_) { + if (info.import) { + blocks.push_back(info.block); + } + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data_preliminary, std::move(blocks), + ig.get_promise()); + } + for (auto &[block_id, info] : blocks_) { + if (info.import) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [promise = ig.get_promise(), block = info.block, manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure(manager, &ValidatorManager::set_block_data, handle, std::move(block), + std::move(promise)); + }); + if (info.proof_link.not_null()) { + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(600.0), + ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); + } + } + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_next_masterchain_block); + } + }); +} + +void ArchiveImporterLocal::apply_next_masterchain_block() { + auto it = masterchain_blocks_.find(last_masterchain_state_->get_seqno() + 1); + if (it == masterchain_blocks_.end()) { + LOG(INFO) << "Applied masterchain blocks, last seqno = " << last_masterchain_state_->get_seqno(); + apply_shard_blocks(); + return; + } + BlockIdExt block_id = it->second; + LOG(DEBUG) << "Applying masterchain block " << block_id.to_str(); + BlockInfo &info = blocks_[block_id]; + run_apply_block_query(block_id, info.block, block_id, manager_, td::Timestamp::in(600.0), + [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db_short, block_id, + [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_next_masterchain_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_next_masterchain_block(td::Ref state) { + last_masterchain_state_ = state; + imported_any_ = true; + LOG(DEBUG) << "Applied masterchain block " << state->get_block_id().to_str(); + apply_next_masterchain_block(); +} + +void ArchiveImporterLocal::apply_shard_blocks() { + if (current_shard_client_seqno_ == new_shard_client_seqno_) { + finish_query(); + return; + } + auto it = shard_configs_.find(current_shard_client_seqno_ + 1); + if (it == shard_configs_.end()) { + abort_query(td::Status::Error("no shard config for the next shard client seqno")); + return; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + BlockIdExt mc_block_id = it->second.first; + LOG(DEBUG) << "Applying top shard blocks from " << current_shard_client_seqno_ + 1; + for (const BlockIdExt &block_id : it->second.second) { + apply_shard_block(block_id, mc_block_id, ig.get_promise()); + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_shard_blocks); + }); +} + +void ArchiveImporterLocal::applied_shard_blocks() { + LOG(DEBUG) << "Applied top shard blocks from " << current_shard_client_seqno_ + 1; + ++current_shard_client_seqno_; + imported_any_ = true; + apply_shard_blocks(); +} + +void ArchiveImporterLocal::apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, + td::Promise promise) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont1, R.move_as_ok(), mc_block_id, + std::move(promise)); + }); +} + +void ArchiveImporterLocal::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont2, handle, mc_block_id, + std::move(promise)); + }; + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + apply_shard_block(handle->one_prev(true), mc_block_id, std::move(promise)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporterLocal::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + td::Ref block = blocks_[handle->id()].block; + CHECK(block.not_null()); + LOG(DEBUG) << "Applying shard block " << handle->id().to_str(); + run_apply_block_query(handle->id(), std::move(block), mc_block_id, manager_, td::Timestamp::in(600.0), + std::move(promise)); +} + +void ArchiveImporterLocal::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + }); +} + +void ArchiveImporterLocal::abort_query(td::Status error) { + if (!imported_any_) { + LOG(ERROR) << "Archive import: " << error; + promise_.set_error(std::move(error)); + stop(); + } else { + LOG(WARNING) << "Archive import: " << error; + finish_query(); + } +} + +void ArchiveImporterLocal::finish_query() { + LOG(WARNING) << "Imported archive in " << perf_timer_.elapsed() + << "s : mc_seqno=" << last_masterchain_state_->get_seqno() + << " shard_seqno=" << current_shard_client_seqno_; + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), current_shard_client_seqno_)}); + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp new file mode 100644 index 000000000..e9e6efdb1 --- /dev/null +++ b/validator/import-db-slice-local.hpp @@ -0,0 +1,103 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/actor/actor.h" +#include "td/utils/port/path.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporterLocal : public td::actor::Actor { + public: + ArchiveImporterLocal(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + td::Status process_package(std::string path); + + void process_masterchain_blocks(); + void process_masterchain_blocks_cont(); + + void import_first_key_block(); + void checked_key_block_proof(BlockHandle handle); + void applied_key_block(td::Ref state); + + void checked_masterchain_proofs(); + void got_shard_client_state(td::Ref state); + + void try_advance_shard_client_seqno(); + void try_advance_shard_client_seqno_cont(td::Ref mc_block); + + void processed_shard_blocks(); + void store_data(); + void apply_next_masterchain_block(); + void applied_next_masterchain_block(td::Ref state); + + void apply_shard_blocks(); + void applied_shard_blocks(); + + void apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string db_root_; + td::Ref last_masterchain_state_; + BlockSeqno shard_client_seqno_; + + td::Ref opts_; + + td::actor::ActorId manager_; + + std::vector to_import_files_; + td::Promise> promise_; + + struct BlockInfo { + td::Ref block; + td::Ref proof; + td::Ref proof_link; + bool import = false; + }; + std::map blocks_; + std::map masterchain_blocks_; + + td::Ref shard_client_state_; + BlockSeqno new_shard_client_seqno_; + BlockSeqno current_shard_client_seqno_; + std::set visited_shard_blocks_; + std::set new_zerostates_; + + std::map>> shard_configs_; + + bool imported_any_ = false; + + td::PerfWarningTimer perf_timer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 06573d347..aafd3c22f 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -45,13 +45,16 @@ ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref , manager_(manager) , to_import_files_(std::move(to_import_files)) , use_imported_files_(!to_import_files_.empty()) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("import-slice", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice", duration); + }) { } void ArchiveImporter::start_up() { if (use_imported_files_) { LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; - for (const std::string& path : to_import_files_) { + for (const std::string &path : to_import_files_) { LOG(INFO) << "Importing file from disk " << path; td::Status S = process_package(path, true); if (S.is_error()) { @@ -85,7 +88,7 @@ void ArchiveImporter::downloaded_mc_archive(std::string path) { void ArchiveImporter::processed_mc_archive() { if (masterchain_blocks_.empty()) { - LOG(DEBUG) << "No masterhchain blocks in archive"; + LOG(DEBUG) << "No masterchain blocks in archive"; last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); checked_all_masterchain_blocks(); return; @@ -268,13 +271,13 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { LOG(DEBUG) << "Applied masterchain block #" << handle->id().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); - td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, + td::actor::send_closure(SelfId, &ArchiveImporter::got_new_masterchain_state, td::Ref(R.move_as_ok())); }); td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } -void ArchiveImporter::got_new_materchain_state(td::Ref state) { +void ArchiveImporter::got_new_masterchain_state(td::Ref state) { last_masterchain_state_ = std::move(state); imported_any_ = true; check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); @@ -297,23 +300,42 @@ void ArchiveImporter::checked_all_masterchain_blocks() { }); } +static bool have_new_shards(td::Ref new_state, td::Ref old_state, + ShardIdFull shard_prefix) { + for (auto &shard : new_state->get_shards()) { + if (!shard_intersects(shard_prefix, shard->shard())) { + continue; + } + auto old_shard = old_state->get_shard_from_config(shard->shard()); + if (old_shard.is_null() || old_shard->top_block_id() != shard->top_block_id()) { + return true; + } + } + return false; +} + void ArchiveImporter::download_shard_archives(td::Ref start_state) { start_state_ = start_state; td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); - LOG(DEBUG) << "Monitor min split = " << monitor_min_split; - // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + LOG(DEBUG) << "Monitor min split = " << monitor_min_split + << (have_shard_blocks_ ? ", shard blocks in the main package" : ", no shard blocks in the main package"); // If masterchain package has shard blocks then it's old archive format, don't need to download shards - if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + if (!have_shard_blocks_ && !use_imported_files_) { for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; if (opts_->need_monitor(shard_prefix, start_state)) { - ++pending_shard_archives_; - LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); - download_shard_archive(shard_prefix); + if (have_new_shards(last_masterchain_state_, start_state_, shard_prefix)) { + ++pending_shard_archives_; + LOG(INFO) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } else { + LOG(INFO) << "Not downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str() + << " : no new shard blocks"; + } } } } else { - LOG(DEBUG) << "Skip downloading shard archives"; + LOG(INFO) << "Skip downloading shard archives"; } if (pending_shard_archives_ == 0) { check_next_shard_client_seqno(shard_client_seqno_ + 1); @@ -418,8 +440,10 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); - td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, - td::Timestamp::in(3600), std::move(P)) + td::actor::create_actor( + "downloadstate", handle->id(), masterchain_block_id, + start_state_->persistent_state_split_depth(handle->id().shard_full().workchain), 2, manager_, + td::Timestamp::in(3600), std::move(P)) .release(); return; } @@ -513,6 +537,7 @@ void ArchiveImporter::abort_query(td::Status error) { td::unlink(f).ignore(); } promise_.set_error(std::move(error)); + stop(); return; } LOG(INFO) << "Archive import: " << error; @@ -524,6 +549,7 @@ void ArchiveImporter::finish_query() { td::unlink(f).ignore(); } if (promise_) { + LOG(INFO) << "Imported archive in " << perf_timer_.elapsed(); promise_.set_value({last_masterchain_state_->get_seqno(), std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 04f22642d..fc295216a 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -44,7 +44,7 @@ class ArchiveImporter : public td::actor::Actor { void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); - void got_new_materchain_state(td::Ref state); + void got_new_masterchain_state(td::Ref state); void checked_all_masterchain_blocks(); void download_shard_archives(td::Ref start_state); @@ -91,6 +91,8 @@ class ArchiveImporter : public td::actor::Actor { bool imported_any_ = false; bool have_shard_blocks_ = false; std::vector files_to_cleanup_; + + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b3..834bbd729 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -20,6 +20,7 @@ #include "ton/ton-types.h" #include "validator/interfaces/block-handle.h" +#include "validator/interfaces/persistent-state.h" #include "validator/interfaces/validator-manager.h" namespace ton { @@ -50,20 +51,29 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; - virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; - virtual void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; + virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; @@ -123,12 +133,14 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; - virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0; + virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) = 0; virtual void add_persistent_state_description(td::Ref desc, td::Promise promise) = 0; virtual void get_persistent_state_descriptions( td::Promise>> promise) = 0; + + virtual void iterate_temp_block_handles(std::function f) = 0; }; } // namespace validator diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index c0aa56106..564b040cf 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -26,17 +26,19 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, bool is_local, td::int32 msg_count = -1) : block_id_(block_id) , state_root_(std::move(state_root)) , block_state_proof_(std::move(block_state_proof)) + , is_local_(is_local) , msg_count_(msg_count) { } BlockIdExt block_id_; Ref state_root_; Ref block_state_proof_; + bool is_local_ = false; td::int32 msg_count_; // -1 - no limit static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h new file mode 100644 index 000000000..36d9ef0bd --- /dev/null +++ b/validator/interfaces/persistent-state.h @@ -0,0 +1,90 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "auto/tl/ton_api.h" +#include "td/utils/Variant.h" +#include "td/utils/overloaded.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" + +namespace ton { + +namespace validator { + +struct UnsplitStateType {}; + +struct SplitAccountStateType { + ShardId effective_shard_id; +}; + +struct SplitPersistentStateType {}; + +using PersistentStateType = td::Variant; + +auto persistent_state_id_from_v1_query(auto const &query) { + auto block = create_tl_block_id(create_block_id(query.block_)); + auto mc_block = create_tl_block_id(create_block_id(query.masterchain_block_)); + return create_tl_object(std::move(block), std::move(mc_block), 0); +} + +auto persistent_state_from_v2_query(auto const &query) { + auto block = create_block_id(query.state_->block_); + auto mc_block = create_block_id(query.state_->masterchain_block_); + ShardId effective_shard = static_cast(query.state_->effective_shard_); + + if (effective_shard == 0 || !shard_is_ancestor(block.shard_full().shard, effective_shard)) { + // The second condition is technically a "protocol" violation but since we don't really validate + // stuff here regardless, let's just map it to an unsplit state. + return std::tuple{block, mc_block, PersistentStateType{UnsplitStateType{}}}; + } + + if (effective_shard == block.shard_full().shard) { + return std::tuple{block, mc_block, PersistentStateType{SplitPersistentStateType{}}}; + } + + CHECK(shard_is_proper_ancestor(block.shard_full().shard, effective_shard)); + return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; +} + +inline ShardId persistent_state_to_effective_shard(ShardIdFull const &shard, PersistentStateType const &type) { + ShardId result = 0; + type.visit(td::overloaded([](UnsplitStateType) {}, + [&](SplitAccountStateType type) { result = type.effective_shard_id; }, + [&](SplitPersistentStateType) { result = shard.shard; })); + return result; +} + +inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { + std::string result; + state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, + [&](SplitAccountStateType type) { + int real_pfx_len = shard_prefix_length(shard.shard); + int effective_pfx_len = shard_prefix_length(type.effective_shard_id); + td::uint64 parts_count = 1 << (effective_pfx_len - real_pfx_len); + td::uint64 part_idx = + type.effective_shard_id >> (64 - effective_pfx_len) & (parts_count - 1); + result = + "part " + std::to_string(part_idx + 1) + " out of " + std::to_string(parts_count); + }, + [&](SplitPersistentStateType) { result = "split header"; })); + return result; +} + +} // namespace validator + +} // namespace ton diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index e88a970bb..6e31cc136 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -35,6 +35,7 @@ class ShardTopBlockDescription : public td::CntObject { virtual bool after_split() const = 0; virtual bool after_merge() const = 0; virtual CatchainSeqno catchain_seqno() const = 0; + virtual const std::vector& get_chain_blocks() const = 0; virtual UnixTime generated_at() const = 0; // if method returns false this shard block description is discarded diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 64aea9b62..9a3a5aab0 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -69,8 +69,9 @@ class MasterchainState : virtual public ShardState { virtual td::Ref get_total_validator_set(int next) const = 0; // next = -1 -> prev, next = 0 -> cur virtual bool rotated_all_shards() const = 0; virtual std::vector> get_shards() const = 0; - virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; + virtual td::Ref get_shard_from_config(ShardIdFull shard, bool exact = true) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; @@ -89,6 +90,7 @@ class MasterchainState : virtual public ShardState { return td::Status::OK(); } virtual block::SizeLimitsConfig::ExtMsgLimits get_ext_msg_limits() const = 0; + virtual block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 00fb77e1e..90e524756 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -30,6 +30,9 @@ #include "crypto/vm/db/DynamicBagOfCellsDb.h" #include "validator-session/validator-session-types.h" #include "auto/tl/lite_api.h" +#include "impl/out-msg-queue-proof.hpp" + +#include namespace ton { @@ -52,14 +55,168 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct StorageStatCacheStats { + td::uint64 small_cnt = 0, small_cells = 0; + td::uint64 hit_cnt = 0, hit_cells = 0; + td::uint64 miss_cnt = 0, miss_cells = 0; + + tl_object_ptr tl() const { + return create_tl_object(small_cnt, small_cells, hit_cnt, hit_cells, + miss_cnt, miss_cells); + } +}; + struct CollationStats { - td::uint32 bytes, gas, lt_delta; - int cat_bytes, cat_gas, cat_lt_delta; + BlockIdExt block_id{workchainInvalid, 0, 0, RootHash::zero(), FileHash::zero()}; + td::Status status = td::Status::OK(); + + td::Bits256 collated_data_hash = td::Bits256::zero(); + CatchainSeqno cc_seqno = 0; + double collated_at = -1.0; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + int attempt = 0; + PublicKeyHash self = PublicKeyHash::zero(); + bool is_validator = false; + td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; + int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; + double total_time = 0.0; + std::string time_stats; + + td::uint32 transactions = 0; + std::vector shard_configuration; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; + + td::uint64 old_out_msg_queue_size = 0; + td::uint64 new_out_msg_queue_size = 0; + td::uint32 msg_queue_cleaned = 0; + struct NeighborStats { + ShardIdFull shard; + bool is_trivial = false; + bool is_local = false; + int msg_limit = -1; + td::uint32 processed_msgs = 0; + td::uint32 skipped_msgs = 0; + bool limit_reached = false; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_shard_id(shard), is_trivial, is_local, msg_limit, processed_msgs, skipped_msgs, limit_reached); + } + }; + std::vector neighbors; + + double load_fraction_queue_cleanup = -1.0; + double load_fraction_dispatch = -1.0; + double load_fraction_internals = -1.0; + double load_fraction_externals = -1.0; + double load_fraction_new_msgs = -1.0; + + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; + td::RealCpuTimer::Time queue_cleanup; + td::RealCpuTimer::Time prelim_storage_stat; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + td::RealCpuTimer::Time final_storage_stat; + td::RealCpuTimer::Time create_block; + td::RealCpuTimer::Time create_collated_data; + td::RealCpuTimer::Time create_block_candidate; + + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " queue_cleanup=" << queue_cleanup.get(is_cpu) + << " prelim_storage_stat=" << prelim_storage_stat.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu) + << " final_storage_stat=" << final_storage_stat.get(is_cpu) + << " create_block=" << create_block.get(is_cpu) + << " create_collated_data=" << create_collated_data.get(is_cpu) + << " create_block_candidate=" << create_block_candidate.get(is_cpu); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; + + tl_object_ptr tl() const { + std::vector> shards_obj; + for (const BlockIdExt& block_id : shard_configuration) { + shards_obj.push_back(create_tl_block_id(block_id)); + } + std::vector> neighbors_obj; + for (const NeighborStats& neighbor : neighbors) { + neighbors_obj.push_back(neighbor.tl()); + } + auto block_stats = create_tl_object( + create_tl_object(ext_msgs_total, ext_msgs_filtered, + ext_msgs_accepted, ext_msgs_rejected), + transactions, std::move(shards_obj), old_out_msg_queue_size, new_out_msg_queue_size, msg_queue_cleaned, + std::move(neighbors_obj)); + return create_tl_object( + create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, + work_time.total.cpu, time_stats, work_time.to_str(false), work_time.to_str(true), + create_tl_object( + estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, + cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, + load_fraction_externals, load_fraction_new_msgs, limits_log), + std::move(block_stats), storage_stat_cache.tl()); + } +}; + +struct ValidationStats { + BlockIdExt block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + double validated_at = -1.0; + PublicKeyHash self = PublicKeyHash::zero(); + bool valid = false; + std::string comment; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + double total_time = 0.0; + std::string time_stats; + + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, + actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, + time_stats, work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); + } +}; + +struct CollatorNodeResponseStats { + PublicKeyHash self = PublicKeyHash::zero(); + PublicKeyHash validator_id = PublicKeyHash::zero(); + double timestamp = -1.0; + BlockIdExt block_id, original_block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + + tl_object_ptr tl() const { + return create_tl_object( + self.bits256_value(), validator_id.bits256_value(), timestamp, create_tl_block_id(block_id), + create_tl_block_id(original_block_id), collated_data_hash); + ; + } }; using ValidateCandidateResult = td::Variant; @@ -70,10 +227,18 @@ class ValidatorManager : public ValidatorManagerInterface { } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; + virtual void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; @@ -104,7 +269,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; @@ -118,8 +283,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void get_external_messages(ShardIdFull shard, td::Promise, int>>> promise) = 0; virtual void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) = 0; - virtual void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) = 0; + virtual void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) = 0; virtual void complete_external_messages(std::vector to_delay, std::vector to_delete) = 0; virtual void complete_ihr_messages(std::vector to_delay, @@ -133,7 +298,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; - virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, + virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) = 0; @@ -145,13 +311,17 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; @@ -160,16 +330,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void try_get_static_file(FileHash file_hash, td::Promise promise) = 0; - virtual void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; virtual void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; - virtual void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) = 0; virtual void archive(BlockHandle handle, td::Promise promise) = 0; @@ -186,9 +347,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; - virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; - virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; - virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; + virtual void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { + } + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + } + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + } virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; @@ -208,14 +372,29 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id, bool success) { } - virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - td::optional stats) { + virtual void log_collate_query_stats(CollationStats stats) { + } + virtual void log_validate_query_stats(ValidationStats stats) { } - virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) { + virtual void log_collator_node_response_stats(CollatorNodeResponseStats stats) { } virtual void add_persistent_state_description(td::Ref desc) = 0; + virtual void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) { + promise.set_error(td::Status::Error("not implemented")); + } + virtual void update_storage_stat_cache(std::vector, td::uint32>> data) { + // not implemented + } + + virtual void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + promise.set_result(td::Unit()); + } + + virtual void iterate_temp_block_handles(std::function f) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b43..eb06dd304 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -64,7 +64,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { UNREACHABLE(); } @@ -128,8 +128,12 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); + run_collate_query(CollateParams{.shard = shard_id, + .min_masterchain_block_id = last_masterchain_block_id_, + .prev = prev, + .creator = created_by, + .validator_set = val_set}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -152,8 +156,13 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -200,14 +209,15 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { @@ -276,7 +286,8 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { if (!last_masterchain_block_handle_) { shard_blocks_raw_.push_back(std::move(data)); return; @@ -315,14 +326,15 @@ void ValidatorManagerImpl::dec_pending_new_blocks() { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -334,14 +346,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -395,7 +407,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -414,7 +426,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -489,7 +501,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -520,15 +532,15 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { if (!last_masterchain_block_handle_) { promise.set_result(std::vector>{}); return; } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } @@ -686,20 +698,37 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::BufferSlice state, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } @@ -786,8 +815,9 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -906,7 +936,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { UNREACHABLE(); } @@ -947,7 +977,7 @@ void ValidatorManagerImpl::update_shard_blocks() { } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555..1c4d00d72 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -96,7 +96,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -108,12 +108,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise) override; + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -133,8 +134,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { @@ -148,17 +150,27 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -186,7 +198,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -200,8 +212,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_external_messages(ShardIdFull shard, td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; @@ -240,8 +252,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); @@ -263,8 +275,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { @@ -341,34 +351,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } @@ -399,15 +384,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, @@ -448,6 +424,18 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override { } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + void update_options(td::Ref opts) override { opts_ = std::move(opts); } diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 91c598aa2..717ab3aa5 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -55,7 +55,9 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { }); LOG(ERROR) << "running collate query"; - run_collate_hardfork(shard_id, block_id, prev, actor_id(this), td::Timestamp::in(10.0), std::move(P)); + run_collate_query( + CollateParams{.shard = shard_id, .min_masterchain_block_id = block_id, .prev = prev, .is_hardfork = true}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::created_candidate(BlockCandidate candidate) { @@ -165,14 +167,15 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -184,14 +187,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -245,7 +248,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -264,7 +267,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -339,7 +342,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -370,8 +373,8 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { } void ValidatorManagerImpl::get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736..37b5d6dd3 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -107,7 +107,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block(ReceivedBlock block, td::Promise promise) override { UNREACHABLE(); } - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override { + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override { UNREACHABLE(); } @@ -128,16 +128,17 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -154,10 +155,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override { UNREACHABLE(); } - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { UNREACHABLE(); } @@ -174,23 +176,40 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override { + UNREACHABLE(); + } + + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override { + UNREACHABLE(); + } + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override { + UNREACHABLE(); + } void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override { + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override { UNREACHABLE(); } void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override { UNREACHABLE(); @@ -228,8 +247,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::BufferSlice data, int mode) override { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, @@ -244,7 +263,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_external_messages(ShardIdFull shard, td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override { @@ -306,8 +325,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override { + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, @@ -333,8 +352,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { @@ -406,34 +423,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { UNREACHABLE(); } @@ -464,15 +456,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, @@ -516,6 +499,18 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override { } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + private: td::Ref opts_; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6f304680b..886a71544 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -59,13 +59,13 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han handle_ = std::move(handle); key_blocks_.push_back(handle_); - if (opts_->initial_sync_disabled()) { + if (opts_->initial_sync_disabled() && handle_->id().seqno() == 0) { status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); return; @@ -74,55 +74,65 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han download_proof_link(); } -void ValidatorManagerMasterchainReiniter::download_proof_link() { +void ValidatorManagerMasterchainReiniter::download_proof_link(bool try_local) { if (handle_->id().id.seqno == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_zero_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + if (try_local) { + LOG(DEBUG) << "failed to get proof link from local import: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + } delay_action( - [SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); }, + [SelfId]() { + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); + }, td::Timestamp::in(1.0)); } else { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, - std::move(P)); + if (try_local) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), + handle_->id(), std::move(P)); + } else { + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + std::move(P)); + } } } -void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice proof) { - auto pp = create_proof_link(handle_->id(), std::move(proof)); - if (pp.is_error()) { - LOG(WARNING) << "bad proof link: " << pp.move_as_error(); +void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice data) { + auto r_proof = create_proof(handle_->id(), std::move(data)); + if (r_proof.is_error()) { + LOG(WARNING) << "bad proof link: " << r_proof.move_as_error(); download_proof_link(); return; } + auto proof = r_proof.move_as_ok(); - auto proof_link = pp.move_as_ok(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); } else { auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); }); - td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof, std::move(P)); } }); - - run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_query(handle_->id(), proof, manager_, td::Timestamp::in(60.0), std::move(P), + /* skip_check_signatures = */ true); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -130,6 +140,10 @@ void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { } void ValidatorManagerMasterchainReiniter::try_download_key_blocks(bool try_start) { + if (opts_->initial_sync_disabled()) { + download_masterchain_state(); + return; + } if (!download_new_key_blocks_until_) { if (opts_->allow_blockchain_init()) { download_new_key_blocks_until_ = td::Timestamp::in(60.0); @@ -183,8 +197,6 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -204,6 +216,11 @@ void ValidatorManagerMasterchainReiniter::got_key_block_handle(td::uint32 idx, B CHECK(!key_blocks_[idx]); CHECK(handle->inited_proof()); CHECK(handle->is_key_block()); + if (idx + 1 == key_blocks_.size()) { + int ago = (int)td::Clocks::system() - (int)handle->unix_time(); + LOG(WARNING) << "last key block is " << handle->id().to_str() << ", " << ago << "s ago"; + status_.set_status(PSTRING() << "last key block is " << handle->id().seqno() << ", " << ago << " s ago"); + } key_blocks_[idx] = std::move(handle); CHECK(pending_ > 0); if (!--pending_) { @@ -263,7 +280,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { R.move_as_ok()); } }); - td::actor::create_actor("downloadstate", block_id_, block_id_, 2, manager_, + td::actor::create_actor("downloadstate", block_id_, block_id_, 0, 2, manager_, td::Timestamp::in(3600 * 3), std::move(P)) .release(); } @@ -331,7 +348,7 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl handle_ = std::move(handle); if (!handle_->received_state()) { LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), true, [SelfId = actor_id(this), handle = handle_](td::Result> R) { td::actor::send_closure( SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, handle); diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 901b826bd..90dc5ccc9 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -44,7 +44,7 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { void start_up() override; void written_hardforks(); void got_masterchain_handle(BlockHandle handle); - void download_proof_link(); + void download_proof_link(bool try_local = true); void downloaded_proof_link(td::BufferSlice data); void downloaded_zero_state(); diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092..a5315788d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -31,6 +31,7 @@ #include "state-serializer.hpp" #include "get-next-key-blocks.h" #include "import-db-slice.hpp" +#include "import-db-slice-local.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -42,6 +43,8 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "db/fileref.hpp" +#include "td/actor/MultiPromise.h" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -210,7 +213,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; @@ -233,6 +236,11 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi } void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { + for (auto &[_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(block_id.shard_full())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, block_id, cc_seqno); + } + } } void ValidatorManagerImpl::sync_complete(td::Promise promise) { @@ -304,21 +312,22 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::int64 offset, td::int64 max_length, + PersistentStateType type, td::int64 offset, td::int64 max_length, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, offset, max_length, - std::move(promise)); + td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, type, offset, + max_length, std::move(promise)); } void ValidatorManagerImpl::get_previous_persistent_state_files( @@ -389,7 +398,8 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { - if (!is_validator()) { + if (!validating_masterchain() && collator_nodes_.empty() && + (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } if (last_masterchain_state_.is_null()) { @@ -471,7 +481,7 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (!is_validator()) { + if (collator_nodes_.empty() && (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } auto R = create_ihr_message(std::move(data)); @@ -487,19 +497,21 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !cached_block_candidates_.count(block_id)) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { + VLOG(VALIDATOR_DEBUG) << "dropping shard block description broadcast: not inited"; return; } - if (!last_masterchain_block_handle_) { - VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + if (!is_validator() && !opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { return; } - if (!started_) { + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block description broadcast"; return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{block_id.shard_full(), cc_seqno}); - if (it != shard_blocks_.end() && block_id.id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && block_id.id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } @@ -514,31 +526,46 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } -void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { - if (!last_masterchain_block_handle_) { +void ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; return; } - if (!started_) { - return; - } if (!need_monitor(block_id.shard_full())) { VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; return; } - add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); + add_cached_block_data(block_id, std::move(data)); } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { + for (const BlockIdExt &block_id : desc->get_chain_blocks()) { + if (cached_checked_shard_block_descriptions_.put(block_id, td::Unit())) { + if (cached_block_data_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for new shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); + } + } + } if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { return; } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::new_shard_block_description, desc); + } + if (!is_validator()) { + return; + } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); if (need_monitor(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { @@ -551,37 +578,121 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), true, std::move(P)); + } + if (validating_masterchain()) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + preload_msg_queue_to_masterchain(desc, ig.get_promise()); + wait_verify_shard_blocks({desc->block_id()}, ig.get_promise().wrap([desc](td::Unit) { + VLOG(VALIDATOR_DEBUG) << "verified top shard block " << desc->block_id().to_str(); + return td::Unit{}; + })); + ig.add_promise([SelfId = actor_id(this), desc](td::Result R) mutable { + if (R.is_error()) { + VLOG(VALIDATOR_DEBUG) << "shard block description: " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::set_shard_block_description_ready, std::move(desc)); + }); + } + for (auto &[_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(desc->shard())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), + desc->catchain_seqno()); + } } } -void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) { - BlockIdExt id = block.id; - if (block.id.is_masterchain()) { +void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc, + td::Promise promise) { + if (!validating_masterchain()) { + promise.set_error(td::Status::Error("not validating masterchain")); return; } - if (cached_block_candidates_.emplace(id, std::move(block)).second) { - cached_block_candidates_lru_.push_back(id); - { - auto it = wait_block_data_.find(id); - if (it != wait_block_data_.end()) { - auto r_block = create_block(cached_block_candidates_[id].clone()); - if (r_block.is_ok()) { - td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + promise.set_error(td::Status::Error("shard block description is outdated")); + return; + } + wait_neighbor_msg_queue_proofs( + ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(10.0), + [=, SelfId = actor_id(this), retry_at = td::Timestamp::in(1.0), + promise = std::move(promise)](td::Result>> R) mutable { + if (R.is_error()) { + delay_action( + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc, + std::move(promise)); + }, + retry_at); + return; } + auto res = R.move_as_ok(); + auto &queue = res[desc->block_id()]; + CHECK(queue.not_null()); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue), + std::move(promise)); + }); +} + +void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Ref desc, + td::Ref res, + td::Promise promise) { + VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id().to_str(); + cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); + promise.set_value(td::Unit()); +} + +void ValidatorManagerImpl::set_shard_block_description_ready(td::Ref desc) { + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (it == shard_blocks_.end()) { + return; + } + auto &info = it->second; + if (info.ready_desc.is_null() || info.ready_desc->block_id().seqno() < desc->block_id().seqno()) { + VLOG(VALIDATOR_DEBUG) << "top shard block description is ready: " << desc->block_id().to_str(); + info.ready_desc = desc; + } +} + +void ValidatorManagerImpl::add_cached_block_data(BlockIdExt block_id, td::BufferSlice data) { + if (block_id.is_masterchain()) { + return; + } + td::BufferSlice& block_data = cached_block_data_.get(block_id); + if (!block_data.empty()) { + return; + } + block_data = std::move(data); + { + auto it = wait_block_data_.find(block_id); + if (it != wait_block_data_.end()) { + auto r_block = create_block(ReceivedBlock{block_id, block_data.clone()}); + if (r_block.is_ok()) { + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); + } else { + LOG(WARNING) << "Failed to parse cached block " << block_id.to_str() << " : " << r_block.move_as_error(); } } - { - auto it = wait_state_.find(id); - if (it != wait_state_.end()) { - // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request - td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); - } + } + { + auto it = wait_state_.find(block_id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); } } - if (cached_block_candidates_lru_.size() > max_cached_candidates()) { - CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); - cached_block_candidates_lru_.pop_front(); + + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for checked shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); } } @@ -684,11 +795,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); - } + bool wait_store, td::Promise> promise) { auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -697,36 +804,116 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); + }); + auto id = + td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P1), + std::move(P2), get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + if (wait_store) { + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + } else if (it->second.preliminary_done_) { + promise.set_result(it->second.preliminary_result_); + return; + } else { + it->second.waiting_preliminary_.emplace_back(timeout, priority, std::move(promise)); + } auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitBlockState::update_timeout, X.first, X.second); } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), priority, timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), priority, timeout, - std::move(promise)); + wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); } +void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, + last_masterchain_state_); + } + if (dst_shard.is_masterchain()) { + // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries + // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} + // Also, use cache. + // This is performed here and not in OutMsgQueueImporter because it's important to use + // cached_msg_queue_to_masterchain_, which is related to the current list of shard block descriptions + class Worker : public td::actor::Actor { + public: + Worker(size_t pending, td::Promise>> promise) + : pending_(pending), promise_(std::move(promise)) { + } + + void start_up() override { + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_result(td::Ref res) { + result_[res->block_id_] = res; + if (--pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + size_t pending_; + td::Promise>> promise_; + std::map> result_; + }; + auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); + for (const BlockIdExt &block : blocks) { + auto it = cached_msg_queue_to_masterchain_.find(block); + if (it != cached_msg_queue_to_masterchain_.end()) { + td::actor::send_closure(worker, &Worker::on_result, it->second); + continue; + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::vector{1, block}, timeout, + [=](td::Result>> R) { + if (R.is_error()) { + td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); + } else { + auto res = R.move_as_ok(); + CHECK(res.size() == 1); + td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); + } + }); + } + return; + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::move(blocks), timeout, std::move(promise)); +} + void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { auto it = wait_block_data_.find(handle->id()); @@ -735,7 +922,8 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), false, std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), + is_shard_collator(handle->id().shard_full()), std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); @@ -762,10 +950,6 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); - } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -779,7 +963,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -798,7 +982,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), priority, timeout, std::move(promise)); @@ -873,7 +1057,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, priority, timeout, std::move(P)); + wait_block_state(handle, priority, timeout, false, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -954,11 +1138,14 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { std::vector> v; for (auto &b : shard_blocks_) { - v.push_back(b.second); + if (b.second.ready_desc.is_null()) { + continue; + } + v.push_back(b.second.ready_desc); } promise.set_value(std::move(v)); } @@ -1058,9 +1245,8 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId } void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - promise.set_result(it->second.data.clone()); + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + promise.set_result(cached->clone()); return; } td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, @@ -1132,37 +1318,51 @@ void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull accoun td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } -void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { - if (R.is_ok()) { - block_state_cache_[handle->id()] = {R.ok(), td::Timestamp::in(30.0)}; - } +void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R, + bool preliminary) { auto it = wait_state_.find(handle->id()); - if (it != wait_state_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else if (it->second.waiting_.size() != 0) { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); - }); - auto id = - td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state_to_download(handle->id())) - .release(); - it->second.actor_ = id; - return; - } - } else { - auto r = R.move_as_ok(); + if (it == wait_state_.end()) { + return; + } + if (R.is_ok()) { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_result(r); + } + it->second.preliminary_done_ = true; + it->second.preliminary_result_ = r; + it->second.waiting_preliminary_.clear(); + if (!preliminary) { + block_state_cache_[handle->id()] = {r, td::Timestamp::in(30.0)}; for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + wait_state_.erase(it); + } + } else if (!preliminary) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_error(S.clone()); + } + wait_state_.erase(it); + } else if (!it->second.waiting_.empty() || !it->second.waiting_preliminary_.empty()) { + auto X = it->second.get_timeout(); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); + }); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P1), std::move(P2), + get_block_persistent_state_to_download(handle->id())) + .release(); + it->second.actor_ = id; } - wait_state_.erase(it); } } @@ -1180,10 +1380,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, false, - std::move(P)) - .release(); + auto id = td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, + is_shard_collator(handle->id().shard_full()), std::move(P)) + .release(); it->second.actor_ = id; return; } @@ -1211,20 +1410,37 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::BufferSlice state, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } @@ -1307,14 +1523,20 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } if (!id.is_masterchain()) { - add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + add_cached_block_data(id, candidate.data.clone()); + } + LOG(INFO) << "Got candidate " << id.to_str() << " with " << candidate.out_msg_queue_proof_broadcasts.size() + << " out msg queue proof broadcasts"; + for (auto broadcast : candidate.out_msg_queue_proof_broadcasts) { + callback_->send_out_msg_queue_proof_broadcast(broadcast); } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -1570,12 +1792,9 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { - { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); - return promise.set_value(it->second.clone()); - } + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); + return promise.set_value(ReceivedBlock{id, cached->clone()}); } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1586,9 +1805,9 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { - callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3), + callback_->download_persistent_state(id, masterchain_block_id, type, priority, td::Timestamp::in(3600 * 3), std::move(promise)); } @@ -1600,14 +1819,13 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { if (!block_id.is_masterchain()) { - auto it = cached_block_candidates_.find(block_id); - if (it != cached_block_candidates_.end()) { - // Proof link can be created from the cached block candidate - LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + if (auto cached = cached_block_data_.get_if_exists(block_id, false)) { + // Proof link can be created from the cached block data + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached block data for " << block_id.to_str(); - TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(*cached), "failed to create proof link: "); - TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(block_id, block_root), "failed to create proof link: "); promise.set_result(std::move(proof_link)); return; @@ -1649,15 +1867,10 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mo callback_->send_broadcast(std::move(broadcast), mode); } -void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, - tl_object_ptr telemetry) { - callback_->send_validator_telemetry(key, std::move(telemetry)); -} - void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { - callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(5.0), std::move(promise)); } @@ -1667,11 +1880,73 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } +void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = to_import_all_.upper_bound(masterchain_block_id.seqno() + 1); + while (true) { + if (it == to_import_all_.begin()) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + --it; + bool stop = false; + for (const std::string &path : it->second) { + td::BufferSlice result; + auto r_package = Package::open(path, false, false); + if (r_package.is_error()) { + LOG(WARNING) << "Cannot open package " << path << " : " << r_package.move_as_error(); + continue; + } + auto package = r_package.move_as_ok(); + package.iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + return true; + } + auto f = F.move_as_ok(); + BlockIdExt id; + bool is_proof = false; + f.ref().visit(td::overloaded( + [&](const fileref::Block &p) { + id = p.block_id; + is_proof = false; + }, + [&](const fileref::Proof &p) { + id = p.block_id; + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + id = p.block_id; + is_proof = true; + }, + [&](const auto &) {})); + if (is_proof && id == block_id) { + result = std::move(data); + return false; + } + if (shard_intersects(id.shard_full(), block_id.shard_full()) && id.seqno() < block_id.seqno()) { + stop = true; + } + return true; + }); + if (!result.empty()) { + promise.set_result(std::move(result)); + return; + } + } + if (block_id.is_masterchain() || stop) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + } +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); + storage_stat_cache_ = td::actor::create_actor("storagestatcache"); td::mkdir(db_root_ + "/tmp/").ensure(); td::mkdir(db_root_ + "/catchains/").ensure(); @@ -1688,6 +1963,7 @@ void ValidatorManagerImpl::start_up() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::started, R.move_as_ok()); }); + size_t to_import_files = 0; auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); @@ -1707,26 +1983,31 @@ void ValidatorManagerImpl::start_up() { } fname = fname.substr(8); - while (fname.size() > 1 && fname[0] == '0') { - fname.remove_prefix(1); - } auto i = fname.find('.'); if (i == td::Slice::npos) { return; } fname = fname.substr(0, i); + while (fname.size() > 1 && fname[0] == '0') { + fname.remove_prefix(1); + } auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } auto seqno = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + LOG(DEBUG) << "found archive slice '" << cfname << "' for seqno " << seqno; to_import_[seqno].push_back(cfname.str()); + ++to_import_files; } }); + to_import_all_ = to_import_; if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } + if (to_import_files > 0) { + LOG(INFO) << "found " << to_import_files << " files to import"; + } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); @@ -1779,7 +2060,6 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { if (opts_->nonfinal_ls_queries_enabled()) { candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); } - init_validator_telemetry(); auto Q = td::PromiseCreator::lambda( [SelfId = actor_id(this)](td::Result>> R) { @@ -1909,9 +2189,15 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, - actor_id(this), std::move(to_import_files), std::move(P)) - .release(); + if (to_import_files.empty()) { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } else { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } } void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { @@ -1971,7 +2257,6 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, last_key_block_handle_->unix_time()); } - init_validator_telemetry(); } update_shard_overlays(); @@ -1982,7 +2267,33 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } - + if (validating_masterchain() || !collator_nodes_.empty()) { + std::set collating_shards; + if (validating_masterchain()) { + collating_shards.emplace(masterchainId); + } + for (const auto &collator : collator_nodes_) { + collating_shards.insert(collator.second.shards.begin(), collator.second.shards.end()); + } + if (!collating_shards.empty()) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::new_masterchain_block_notification, + last_masterchain_state_, std::move(collating_shards)); + } + } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); + } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_masterchain_state, + last_masterchain_state_); + } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::update_masterchain_state, last_masterchain_state_); + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; } @@ -1993,9 +2304,9 @@ void ValidatorManagerImpl::update_shard_overlays() { std::set shards_to_monitor; shards_to_monitor.insert(ShardIdFull{masterchainId}); std::set workchains; - for (const auto& shard : last_masterchain_state_->get_shards()) { + for (const auto &shard : last_masterchain_state_->get_shards()) { workchains.insert(shard->shard().workchain); - if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + if (opts_->need_monitor(shard->shard(), last_masterchain_state_)) { shards_to_monitor.insert(shard->shard()); } } @@ -2019,7 +2330,7 @@ void ValidatorManagerImpl::update_shards() { td::uint32 threshold = 9407194; bool force_group_id_upgrade = last_masterchain_seqno_ == threshold; auto legacy_opts_hash = opts.get_hash(); - if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() + if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); @@ -2110,7 +2421,6 @@ void ValidatorManagerImpl::update_shards() { auto legacy_val_group_id = get_validator_set_id(shard, val_set, legacy_opts_hash, key_seqno, opts); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); - auto it = validator_groups_.find(legacy_val_group_id); if (it != validator_groups_.end()) { new_validator_groups_.emplace(val_group_id, std::move(it->second)); @@ -2134,6 +2444,7 @@ void ValidatorManagerImpl::update_shards() { } active_validator_groups_master_ = active_validator_groups_shard_ = 0; + adnl::AdnlNodeIdShort mc_validator_adnl_id = adnl::AdnlNodeIdShort::zero(); if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2185,6 +2496,12 @@ void ValidatorManagerImpl::update_shards() { new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } } + if (shard.is_masterchain()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } } @@ -2206,6 +2523,12 @@ void ValidatorManagerImpl::update_shards() { val_group_id, ValidatorGroupEntry{ create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } + if (shard.is_masterchain() && mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } @@ -2248,6 +2571,7 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(serializer_, &AsyncStateSerializer::auto_disable_serializer, is_validator() && last_masterchain_state_->get_global_id() == -239); // mainnet only } + init_shard_block_verifier(mc_validator_adnl_id); } void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { @@ -2257,29 +2581,36 @@ void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vectorsecond; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - shard_blocks_.erase(it2); - } else { - ++it; - } + for (auto it = shard_blocks_.begin(); it != shard_blocks_.end();) { + auto &B = it->second; + if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + it = shard_blocks_.erase(it); + continue; } + if (B.ready_desc.not_null() && + !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + B.ready_desc = {}; + } + ++it; } - { - auto it = out_shard_blocks_.begin(); - while (it != out_shard_blocks_.end()) { - auto &B = it->second; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - out_shard_blocks_.erase(it2); - } else { - ++it; - } + for (auto it = cached_msg_queue_to_masterchain_.begin(); it != cached_msg_queue_to_masterchain_.end();) { + const BlockIdExt &block_id = it->first; + auto shard = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + if (shard.is_null() || shard->top_block_id().seqno() >= block_id.seqno()) { + it = cached_msg_queue_to_masterchain_.erase(it); + continue; + } + ++it; + } + + for (auto it = out_shard_blocks_.begin(); it != out_shard_blocks_.end();) { + auto &B = it->second; + if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + auto it2 = it++; + out_shard_blocks_.erase(it2); + } else { + ++it; } } } @@ -2321,15 +2652,27 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); + auto descr = validator_set->get_validator(validator_id.bits256_value()); + CHECK(descr); + auto adnl_id = adnl::AdnlNodeIdShort{ + descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; auto G = td::actor::create_actor( PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, - keyring_, adnl_, rldp_, - overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); + keyring_, adnl_, rldp_, rldp2_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), + init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + opts_->need_monitor(shard, last_masterchain_state_)); return G; } } +td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { + auto &actor = collation_managers_[adnl_id]; + if (actor.empty()) { + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), adnl_, rldp2_); + } + return actor.get(); +} + void ValidatorManagerImpl::add_handle_to_lru(BlockHandle handle) { auto it = handle_lru_map_.find(handle->id()); if (it != handle_lru_map_.end()) { @@ -2374,7 +2717,7 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { gc_masterchain_handle_->id().id.seqno < last_masterchain_state_->last_key_block_id().seqno() && gc_masterchain_handle_->id().id.seqno < min_confirmed_masterchain_seqno_ && gc_masterchain_handle_->id().id.seqno < state_serializer_masterchain_seqno_ && - gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { + (double)gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { gc_advancing_ = true; auto block_id = gc_masterchain_handle_->one_next(true); @@ -2386,100 +2729,6 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { } } -void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() == 0) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() >= gc_masterchain_handle_->id().seqno()) { - promise.set_result(false); - return; - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - auto handle = R.move_as_ok(); - CHECK(handle->is_key_block()); - promise.set_result(ValidatorManager::persistent_state_ttl(handle->unix_time()) < td::Clocks::system()); - }); - get_block_handle(masterchain_block_id, false, std::move(P)); -} - -void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise) { - /*if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (!block_id.is_masterchain()) { - if (!gc_masterchain_state_->workchain_is_active(block_id.id.workchain)) { - promise.set_result(false); - return; - } - bool found = false; - auto S = gc_masterchain_state_->get_shard_from_config(block_id.shard_full()); - if (S.not_null()) { - if (block_id.id.seqno >= S->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } else { - auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { - if (shard_intersects(shard->shard(), block_id.shard_full())) { - if (block_id.id.seqno >= shard->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } - } - } - CHECK(found); - } else { - if (block_id.id.seqno >= gc_masterchain_handle_->id().id.seqno) { - promise.set_result(false); - return; - } - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(true); - } - }); - td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));*/ - promise.set_result(false); -} - -void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise promise) { - auto key_ttl = td::Clocks::system() - opts_->key_proof_ttl(); - auto ttl = td::Clocks::system() - opts_->archive_ttl(); - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise), ttl, key_ttl](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(true); - return; - } - auto handle = R.move_as_ok(); - if (!handle->inited_unix_time()) { - promise.set_result(true); - return; - } - if (!handle->inited_is_key_block() || !handle->is_key_block()) { - promise.set_result(handle->unix_time() <= ttl); - } else { - promise.set_result(handle->unix_time() <= key_ttl); - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise promise) { if (!gc_masterchain_handle_) { promise.set_result(false); @@ -2508,27 +2757,6 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise UNREACHABLE(); } -void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(false); - } else { - auto handle = R.move_as_ok(); - if (!handle->moved_to_archive() || !handle->is_applied()) { - promise.set_result(false); - } else { - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - promise.set_result(true); - }); - td::actor::send_closure(db, &Db::store_block_handle, handle, std::move(P)); - } - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { CHECK(gc_advancing_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -2544,7 +2772,7 @@ void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_state, std::move(handle), td::Ref{R.move_as_ok()}); }); - wait_block_state(handle, 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state(handle, 0, td::Timestamp::in(60.0), true, std::move(P)); } void ValidatorManagerImpl::got_next_gc_masterchain_state(BlockHandle handle, td::Ref state) { @@ -2577,6 +2805,9 @@ void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, last_liteserver_state_ = std::move(state); } } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } shard_client_update(seqno); promise.set_value(td::Unit()); } @@ -2610,7 +2841,7 @@ void ValidatorManagerImpl::alarm() { alarm_timestamp() = td::Timestamp::in(1.0); if (shard_client_handle_ && gc_masterchain_handle_) { td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), - static_cast(opts_->archive_ttl())); + opts_->archive_ttl()); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { @@ -2714,7 +2945,7 @@ void ValidatorManagerImpl::alarm() { } alarm_timestamp().relax(log_ls_stats_at_); if (cleanup_mempool_at_.is_in_past()) { - if (is_validator()) { + if (is_validator() || !collator_nodes_.empty()) { get_external_messages(ShardIdFull{masterchainId, shardIdAll}, [](td::Result, int>>>) {}); get_external_messages(ShardIdFull{basechainId, shardIdAll}, @@ -2782,6 +3013,23 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Refget_collators_list()->self_collate; +} + +bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const { + return std::any_of(shards.begin(), shards.end(), + [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); +} + void ValidatorManagerImpl::got_next_key_blocks(std::vector r) { if (r.size() == 0) { delay_action([SelfId = actor_id( @@ -2847,9 +3095,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promise R) mutable { @@ -2891,7 +3138,7 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); - if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + if (is_validator() && last_masterchain_state_.not_null() && last_masterchain_state_->get_global_id() == -239) { serializer_enabled = false; } vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); @@ -2948,112 +3195,17 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise)); } -void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, - validatorsession::ValidatorSessionStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - - std::vector> rounds; - for (const auto &round : stats.rounds) { - std::vector> producers; - for (const auto &producer : round.producers) { - BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; - auto it = recorded_block_stats_.find(cur_block_id); - tl_object_ptr collation_stats; - if (it != recorded_block_stats_.end() && it->second.collator_stats_) { - auto &stats = it->second.collator_stats_.value(); - collation_stats = create_tl_object( - stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, - stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, - stats.ext_msgs_rejected); - } - std::string approvers, signers; - for (bool x : producer.approvers) { - approvers += (x ? '1' : '0'); - } - for (bool x : producer.signers) { - signers += (x ? '1' : '0'); - } - producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, - producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, - producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), - producer.validation_time, producer.validated_at, producer.validation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, - producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), - producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), - producer.serialize_time, producer.deserialize_time, producer.serialized_size)); - } - rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); - } - - auto obj = create_tl_object( - stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id, - stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, - stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, - std::move(rounds)); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - - LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); +void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { + stats.fix_block_ids(); + write_session_stats(stats); } void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back( - create_tl_object(node.id.bits256_value(), node.weight)); - } - auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, - stats.timestamp, stats.self_idx, std::move(nodes)); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - - LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() - << " cc_seqno=" << stats.cc_seqno; + write_session_stats(stats); } void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back(create_tl_object(node.id.bits256_value(), - node.catchain_blocks)); - } - auto obj = create_tl_object(stats.session_id, stats.timestamp, - std::move(nodes)); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - - LOG(INFO) << "Writing end validator group stats for " << stats.session_id; + write_session_stats(stats); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -3324,18 +3476,131 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!serializer_.empty()) { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); } + if (!out_msg_queue_importer_.empty()) { + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); + } if (!queue_size_counter_.empty()) { td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); } for (auto &group : validator_groups_) { - td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); } for (auto &group : next_validator_groups_) { - td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); + } + for (auto &collator : collator_nodes_) { + td::actor::send_closure(collator.second.actor, &CollatorNode::update_options, opts); + } + for (auto &[_, c] : collation_managers_) { + td::actor::send_closure(c, &CollationManager::update_options, opts); + } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); } opts_ = std::move(opts); } +void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + if (shard.is_masterchain() || !shard.is_valid_ext()) { + LOG(WARNING) << "cannot collate shard " << shard.to_str(); + return; + } + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + it = collator_nodes_.emplace(id, Collator()).first; + it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp2_); + if (last_masterchain_state_.not_null()) { + td::actor::send_closure(it->second.actor, &CollatorNode::new_masterchain_block_notification, + last_masterchain_state_); + } + if (shard_client_handle_) { + td::actor::send_closure(it->second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } + } + if (!it->second.shards.insert(shard).second) { + return; + } + td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); +} + +void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + return; + } + if (!it->second.shards.erase(shard)) { + return; + } + if (it->second.shards.empty()) { + collator_nodes_.erase(it); + } else { + td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); + } +} + +void ValidatorManagerImpl::get_collation_manager_stats( + td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise> promise) + : promise_(std::move(promise)) { + } + + void got_stats(tl_object_ptr s) { + result_.push_back(std::move(s)); + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(create_tl_object(std::move(result_))); + stop(); + } + } + + private: + td::Promise> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("stats", std::move(promise)).release(); + + for (auto &[_, actor] : collation_managers_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure( + actor, &CollationManager::get_stats, + [callback](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + +void ValidatorManagerImpl::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + if (is_shard_collator(dst_shard)) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::add_out_msg_queue_proof, dst_shard, + std::move(proof)); + } else { + VLOG(VALIDATOR_DEBUG) << "Dropping unneeded out msg queue proof to shard " << dst_shard.to_str(); + } +} + void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { @@ -3346,7 +3611,7 @@ void ValidatorManagerImpl::add_persistent_state_description(td::Refsecond; if (prev_desc->end_time <= now) { - for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + for (auto const &[block_id, _] : prev_desc->shard_blocks) { persistent_state_blocks_.erase(block_id); } it = persistent_state_descriptions_.erase(it); @@ -3363,7 +3628,7 @@ void ValidatorManagerImpl::add_persistent_state_description_impl(td::Refmasterchain_id.to_str() << " start_time=" << desc->start_time << " end_time=" << desc->end_time; - for (const BlockIdExt &block_id : desc->shard_blocks) { + for (auto const &[block_id, _] : desc->shard_blocks) { persistent_state_blocks_[block_id] = desc; LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); } @@ -3392,46 +3657,32 @@ td::Ref ValidatorManagerImpl::get_block_persistent_s td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays) { return td::actor::create_actor("manager", std::move(opts), db_root, keyring, adnl, - rldp, overlays); + rldp, rldp2, overlays); } -void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - td::optional stats) { - if (!stats) { - ++(block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_); - return; +void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { + if (stats.status.is_ok()) { + ++(stats.block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_); + write_session_stats(stats); + } else { + ++(stats.block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_); } - auto &record = new_block_stats_record(block_id); - record.collator_work_time_ = work_time; - record.collator_cpu_work_time_ = cpu_work_time; - record.collator_stats_ = std::move(stats.value()); - ++(block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_); } -void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - bool success) { - auto &record = new_block_stats_record(block_id); - record.validator_work_time_ = work_time; - record.validator_cpu_work_time_ = cpu_work_time; - if (success) { - ++(block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_); +void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) { + if (stats.valid) { + ++(stats.block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_); } else { - ++(block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_); + ++(stats.block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_); } + write_session_stats(stats); } -ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { - if (!recorded_block_stats_.count(block_id)) { - recorded_block_stats_lru_.push(block_id); - if (recorded_block_stats_lru_.size() > 4096) { - recorded_block_stats_.erase(recorded_block_stats_lru_.front()); - recorded_block_stats_lru_.pop(); - } - } - return recorded_block_stats_[block_id]; +void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) { + write_session_stats(stats); } void ValidatorManagerImpl::register_stats_provider( @@ -3467,41 +3718,51 @@ void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { } } -void ValidatorManagerImpl::init_validator_telemetry() { - if (last_masterchain_state_.is_null()) { - return; - } - td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); - if (validator_set.is_null()) { - validator_telemetry_.clear(); +template +void ValidatorManagerImpl::write_session_stats(const T &obj) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { return; } - std::set processed; - for (auto& key : temp_keys_) { - if (const ValidatorDescr* desc = validator_set->get_validator(key.bits256_value())) { - processed.insert(key); - adnl::AdnlNodeIdShort adnl_id; - if (desc->addr.is_zero()) { - adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; - } else { - adnl_id = adnl::AdnlNodeIdShort{desc->addr}; - } - auto& telemetry = validator_telemetry_[key]; - if (telemetry.empty()) { - telemetry = td::actor::create_actor( - "telemetry", key, adnl_id, opts_->zero_block_id().file_hash, actor_id(this)); - } - } - } - for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { - if (processed.contains(it->first)) { - ++it; + auto s = td::json_encode(td::ToJson(*obj.tl()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); +} + +void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { + if (local_id != shard_block_verifier_local_id_) { + shard_block_verifier_local_id_ = local_id; + if (local_id.is_zero()) { + shard_block_verifier_ = {}; } else { - it = validator_telemetry_.erase(it); + shard_block_verifier_ = td::actor::create_actor( + "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); } } } +void ValidatorManagerImpl::wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + if (shard_block_verifier_.empty()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "shard block verifier not inited")); + return; + } + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::wait_shard_blocks, std::move(blocks), + std::move(promise)); +} + +void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + shard_block_retainers_[id] = td::actor::create_actor( + "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); +} + +void ValidatorManagerImpl::iterate_temp_block_handles(std::function f) { + td::actor::send_closure(db_, &Db::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/manager.h b/validator/manager.h index 4fe2037fc..652b77588 100644 --- a/validator/manager.h +++ b/validator/manager.h @@ -20,6 +20,7 @@ #include "validator/validator.h" #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" namespace ton { @@ -32,6 +33,7 @@ class ValidatorManagerFactory { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId overlays); }; diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb350..b3fdbc344 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -32,10 +32,15 @@ #include "manager-init.h" #include "state-serializer.hpp" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" -#include "validator-telemetry.hpp" +#include "storage-stat-cache.hpp" #include "impl/candidates-buffer.hpp" +#include "collator-node/collator-node.hpp" +#include "shard-block-verifier.hpp" +#include "shard-block-retainer.hpp" +#include "td/utils/LRUCache.h" #include #include @@ -160,9 +165,17 @@ class ValidatorManagerImpl : public ValidatorManager { WaitList() = default; std::pair get_timeout() const { + return get_timeout_impl(waiting_); + } + void check_timers() { + check_timers_impl(waiting_); + } + + protected: + static std::pair get_timeout_impl(const std::vector> &waiting) { td::Timestamp t = td::Timestamp::now(); td::uint32 prio = 0; - for (auto &v : waiting_) { + for (auto &v : waiting) { if (v.timeout.at() > t.at()) { t = v.timeout; } @@ -172,10 +185,10 @@ class ValidatorManagerImpl : public ValidatorManager { } return {td::Timestamp::at(t.at() + 10.0), prio}; } - void check_timers() { + static void check_timers_impl(std::vector> &waiting) { td::uint32 j = 0; - auto f = waiting_.begin(); - auto t = waiting_.end(); + auto f = waiting.begin(); + auto t = waiting.end(); while (f < t) { if (f->timeout.is_in_past()) { f->promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -186,16 +199,26 @@ class ValidatorManagerImpl : public ValidatorManager { j++; } } - waiting_.resize(j); + waiting.resize(j); } }; template - struct WaitListCaching : public WaitList { - bool done_ = false; - ResType result_; - td::Timestamp remove_at_; + struct WaitListPreliminary : WaitList { + std::vector> waiting_preliminary_; + bool preliminary_done_ = false; + ResType preliminary_result_; + + std::pair get_timeout() const { + auto t1 = WaitList::get_timeout_impl(this->waiting_); + auto t2 = WaitList::get_timeout_impl(waiting_preliminary_); + return {std::max(t1.first, t2.first), std::max(t1.second, t2.second)}; + } + void check_timers() { + WaitList::check_timers_impl(this->waiting_); + WaitList::check_timers_impl(waiting_preliminary_); + } }; - std::map>> wait_state_; + std::map>> wait_state_; std::map>> wait_block_data_; struct CachedBlockState { @@ -209,6 +232,8 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map wait_block_handle_; + td::actor::ActorOwn out_msg_queue_importer_; + private: // HANDLES CACHE std::map> handles_; @@ -232,10 +257,19 @@ class ValidatorManagerImpl : public ValidatorManager { } }; // DATA FOR COLLATOR - std::map> shard_blocks_; + // Shard block will not be used until it is confirmed by trusted nodes (see ShardBlockVerifier) and + // msg queue to masterchain is ready (to avoid too long masterchain collation) + // latest_desc - latest known block + // ready_desc - block ready to be used (may be null) + struct ShardTopBlock { + td::Ref latest_desc; + td::Ref ready_desc; + }; + std::map shard_blocks_; + std::map> cached_msg_queue_to_masterchain_; - std::map cached_block_candidates_; - std::list cached_block_candidates_lru_; + td::LRUCache cached_block_data_{/* max_size = */ 128}; + td::LRUCache cached_checked_shard_block_descriptions_{/* max_size = */ 1024}; struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; @@ -273,12 +307,15 @@ class ValidatorManagerImpl : public ValidatorManager { td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); + td::actor::ActorId get_collation_manager(adnl::AdnlNodeIdShort adnl_id); + struct ValidatorGroupEntry { td::actor::ActorOwn actor; ShardIdFull shard; }; std::map validator_groups_; std::map next_validator_groups_; + std::map> collation_managers_; std::set check_gc_list_; std::vector gc_list_; @@ -346,7 +383,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -355,7 +391,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -366,7 +401,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; @@ -377,12 +412,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise) override; + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; @@ -396,8 +432,9 @@ class ValidatorManagerImpl : public ValidatorManager { void check_external_message(td::BufferSlice data, td::Promise> promise) override; void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -407,17 +444,24 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override; void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -445,7 +489,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -459,8 +503,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_external_messages(ShardIdFull shard, td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; @@ -499,8 +543,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override; void send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, @@ -511,13 +555,15 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -545,11 +591,15 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); - void add_cached_block_candidate(ReceivedBlock block); + void add_cached_block_data(BlockIdExt block_id, td::BufferSlice data); + void preload_msg_queue_to_masterchain(td::Ref desc, td::Promise promise); + void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res, + td::Promise promise); + void set_shard_block_description_ready(td::Ref desc); void register_block_handle(BlockHandle handle); - void finished_wait_state(BlockHandle handle, td::Result> R); + void finished_wait_state(BlockHandle handle, td::Result> R, bool preliminary); void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; @@ -560,38 +610,23 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); + bool is_shard_collator(ShardIdFull shard); ValidatorManagerImpl(td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays) - : opts_(std::move(opts)), db_root_(db_root), keyring_(keyring), adnl_(adnl), rldp_(rldp), overlays_(overlays) { + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays) + : opts_(std::move(opts)) + , db_root_(db_root) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) { } public: - void allow_delete(BlockIdExt block_id, td::Promise promise); - void allow_archive(BlockIdExt block_id, td::Promise promise); - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override; - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - allow_block_state_gc(block_id, std::move(promise)); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override; void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } @@ -611,7 +646,7 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; @@ -619,6 +654,13 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override; + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) override; + + void get_collation_manager_stats( + td::Promise> promise) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { @@ -660,6 +702,13 @@ class ValidatorManagerImpl : public ValidatorManager { ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } + void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::get_cache, std::move(promise)); + } + void update_storage_stat_cache(std::vector, td::uint32>> data) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::update, std::move(data)); + } + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; @@ -698,11 +747,13 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorOwn serializer_; std::map> to_import_; + std::map> to_import_all_; private: std::unique_ptr callback_; @@ -716,15 +767,9 @@ class ValidatorManagerImpl : public ValidatorManager { double state_ttl() const { return opts_->state_ttl(); } - double block_ttl() const { - return opts_->block_ttl(); - } double max_mempool_num() const { return opts_->max_mempool_num(); } - size_t max_cached_candidates() const { - return 128; - } static double max_ext_msg_per_addr_time_window() { return 10.0; } @@ -760,29 +805,28 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn candidates_buffer_; - struct RecordedBlockStats { - double collator_work_time_ = -1.0; - double collator_cpu_work_time_ = -1.0; - td::optional collator_stats_; - double validator_work_time_ = -1.0; - double validator_cpu_work_time_ = -1.0; - }; - std::map recorded_block_stats_; - std::queue recorded_block_stats_lru_; - - void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - td::optional stats) override; - void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) override; - RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + void log_collate_query_stats(CollationStats stats) override; + void log_validate_query_stats(ValidationStats stats) override; + void log_collator_node_response_stats(CollatorNodeResponseStats stats) override; void register_stats_provider( td::uint64 idx, std::string prefix, std::function>>)> callback) override; void unregister_stats_provider(td::uint64 idx) override; - std::map> validator_telemetry_; + void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) override; + + void add_shard_block_retainer(adnl::AdnlNodeIdShort id) override; + + void iterate_temp_block_handles(std::function f) override; - void init_validator_telemetry(); + struct Collator { + td::actor::ActorOwn actor; + std::set shards; + + bool can_collate_shard(ShardIdFull shard) const; + }; + std::map collator_nodes_; std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; @@ -790,6 +834,17 @@ class ValidatorManagerImpl : public ValidatorManager { std::map>>)>>> stats_providers_; + + td::actor::ActorOwn storage_stat_cache_; + + template + void write_session_stats(const T &obj); + + td::actor::ActorOwn shard_block_verifier_; + adnl::AdnlNodeIdShort shard_block_verifier_local_id_ = adnl::AdnlNodeIdShort::zero(); + std::map> shard_block_retainers_; + + void init_shard_block_verifier(adnl::AdnlNodeIdShort local_id); }; } // namespace validator diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5f..da6363f0d 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -28,15 +28,17 @@ namespace validator { namespace fullnode { -DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, - td::uint32 priority, td::Timestamp timeout, +DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , type_(type) + , effective_shard_(persistent_state_to_effective_shard(block_id_.shard_full(), type)) , local_id_(local_id) , overlay_id_(overlay_id) , download_from_(download_from) @@ -48,6 +50,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i , adnl_(adnl) , client_(client) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || effective_shard_ == 0); } void DownloadState::abort_query(td::Status reason) { @@ -73,16 +76,20 @@ void DownloadState::start_up() { status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, - [SelfId = actor_id(this), block_id = block_id_](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::get_block_handle); - } else { - LOG(WARNING) << "got block state from disk: " << block_id.to_str(); - td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); - } - }); + td::Promise P = [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }; + if (block_id_.seqno() == 0) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id_, std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, type_, std::move(P)); + } } void DownloadState::get_block_handle() { @@ -124,22 +131,39 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; - LOG(WARNING) << "downloading state " << block_id_.to_str() << " from " << download_from_; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " (" + << persistent_state_type_to_string(block_id_.shard_full(), type_) << ") from " << download_from_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + td::Promise P; + td::BufferSlice query; + + if (effective_shard_ == 0) { + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + } + }); + + if (masterchain_block_id_.is_valid()) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + query = create_serialize_tl_object(create_tl_block_id(block_id_)); } - }); - - td::BufferSlice query; - if (masterchain_block_id_.is_valid()) { - query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - query = create_serialize_tl_object(create_tl_block_id(block_id_)); + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_state_size, R.move_as_ok()); + } + }); + + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); } if (client_.empty()) { @@ -168,6 +192,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { }, [&, self = this](ton_api::tonNode_preparedState &f) { if (masterchain_block_id_.is_valid()) { + request_total_size(); got_block_state_part(td::BufferSlice{}, 0); return; } @@ -190,8 +215,63 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { create_serialize_tl_object_suffix(std::move(query)), td::Timestamp::in(3.0), std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : download started"); })); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); +} + +void DownloadState::got_state_size(td::BufferSlice size_or_not_found) { + auto F = fetch_tl_object(std::move(size_or_not_found), true); + if (F.is_error()) { + abort_query(F.move_as_error()); + return; + } + prev_logged_timer_ = td::Timer(); + + ton_api::downcast_call(*F.move_as_ok().get(), + td::overloaded( + [&](ton_api::tonNode_persistentStateSizeNotFound &f) { + abort_query(td::Status::Error(ErrorCode::notready, "state not found")); + }, + [&](ton_api::tonNode_persistentStateSize &f) { + total_size_ = f.size_; + got_block_state_part(td::BufferSlice{}, 0); + })); +} + +void DownloadState::request_total_size() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + auto res = fetch_tl_object(R.move_as_ok(), true); + if (res.is_error()) { + return; + } + td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); + }); + + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); + } + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), + FullNode::max_state_size(), rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get size", + create_serialize_tl_object_suffix(std::move(query)), + td::Timestamp::in(3.0), std::move(P)); + } +} + +void DownloadState::got_total_size(td::uint64 size) { + total_size_ = size; } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -203,10 +283,22 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); - LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size(speed) << "/s)"; - status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) - << "/s"); + td::StringBuilder sb; + sb << td::format::as_size(sum_); + if (total_size_) { + sb << "/" << td::format::as_size(total_size_); + } + sb << " (" << td::format::as_size(speed) << "/s"; + if (total_size_) { + sb << ", " << td::StringBuilder::FixedDouble((double)sum_ / (double)total_size_ * 100.0, 2) << "%"; + if (speed > 0 && total_size_ >= sum_) { + td::uint64 rem = (total_size_ - sum_) / speed; + sb << ", " << rem << "s remaining"; + } + } + sb << ")"; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " : " << sb.as_cslice(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sb.as_cslice()); prev_logged_sum_ = sum_; } @@ -233,8 +325,16 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques } }); - td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_), + sum_, part_size); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, "download state", std::move(P), td::Timestamp::in(20.0), std::move(query), diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 470c54318..ebf119bd3 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -33,9 +33,10 @@ namespace fullnode { class DownloadState : public td::actor::Actor { public: - DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, - td::Timestamp timeout, td::actor::ActorId validator_manager, + DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, + td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise); @@ -49,12 +50,17 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void got_state_size(td::BufferSlice size_or_not_found); + void request_total_size(); + void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); void got_block_state(td::BufferSlice data); private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + PersistentStateType type_; + ShardId effective_shard_; adnl::AdnlNodeIdShort local_id_; overlay::OverlayIdShort overlay_id_; @@ -77,6 +83,7 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + td::uint64 total_size_ = 0; ProcessStatus status_; }; diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4fe55ae31..487bdefc8 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -112,7 +112,7 @@ void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_who } BlockHandle handle = R.move_as_ok(); td::actor::send_closure( - manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), false, [SelfId, handle](td::Result> R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, handle->id(), @@ -159,7 +159,7 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); @@ -213,7 +213,7 @@ void QueueSizeCounter::process_top_shard_blocks() { return; } td::actor::send_closure( - manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), false, [=](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Failed to get masterchain state: " << R.move_as_error(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp new file mode 100644 index 000000000..8ef4d8f21 --- /dev/null +++ b/validator/shard-block-retainer.cpp @@ -0,0 +1,201 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-retainer.hpp" +#include + +namespace ton::validator { + +void ShardBlockRetainer::start_up() { + if (last_masterchain_state_.not_null()) { + update_masterchain_state(last_masterchain_state_); + } + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &ShardBlockRetainer::process_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); +} + +void ShardBlockRetainer::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID)); +} + +void ShardBlockRetainer::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = state; + for (auto it = confirmed_blocks_.begin(); it != confirmed_blocks_.end();) { + if (is_block_outdated(*it)) { + it = confirmed_blocks_.erase(it); + } else { + ++it; + } + } + if (last_masterchain_state_->is_key_state() || !inited_) { + validator_adnl_ids_.clear(); + for (int next = 0; next <= 1; ++next) { + auto vset = last_masterchain_state_->get_total_validator_set(next); + if (vset.is_null()) { + continue; + } + for (auto& val : vset->export_vector()) { + adnl::AdnlNodeIdShort adnl_id{val.addr}; + if (adnl_id.is_zero()) { + adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{val.key}.short_id()}; + validator_adnl_ids_.insert(adnl_id); + } + } + } + LOG(INFO) << "Updating validator set: " << validator_adnl_ids_.size() << " adnl ids"; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (!validator_adnl_ids_.contains(it->first.first)) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (not a validator)"; + it = subscribers_.erase(it); + continue; + } + ++it; + } + } + if (!inited_) { + delay_action( + [SelfId = actor_id(this), manager = manager_]() { + td::actor::send_closure( + manager, &ValidatorManager::iterate_temp_block_handles, [SelfId](const BlockHandleInterface& handle) { + if (!handle.id().is_masterchain() && handle.received_state()) { + td::actor::send_closure(SelfId, &ShardBlockRetainer::got_block_from_db, handle.id()); + } + }); + }, + td::Timestamp::in(1.0)); + } + inited_ = true; +} + +void ShardBlockRetainer::new_shard_block_description(td::Ref desc) { + if (last_masterchain_state_.is_null() || is_block_outdated(desc->block_id()) || + !opts_->need_monitor(desc->shard(), last_masterchain_state_)) { + return; + } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), true, + [SelfId = actor_id(this), desc](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); + td::actor::send_closure(SelfId, &ShardBlockRetainer::new_shard_block_description, std::move(desc)); + return; + } + td::actor::send_closure(SelfId, &ShardBlockRetainer::confirm_shard_block_description, std::move(desc)); + }); +} + +void ShardBlockRetainer::process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)) + ShardIdFull shard = create_shard_id(query->shard_); + if (!shard.is_valid_ext() || shard.is_masterchain()) { + promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error(PSTRING() << "unauthorized src " << src)); + return; + } + td::Timestamp& ttl = subscribers_[{src, shard}]; + if (!ttl) { + std::vector blocks; + for (const BlockIdExt& block : confirmed_blocks_) { + if (shard_intersects(block.shard_full(), shard)) { + blocks.push_back(block); + } + } + LOG(INFO) << "New subscriber " << src << " for " << shard.to_str() << ", sending " << blocks.size() << " blocks"; + send_confirmations(src, std::move(blocks)); + } + ttl = td::Timestamp::in(SUBSCRIPTION_TTL); + promise.set_value(create_serialize_tl_object(0)); +} + +void ShardBlockRetainer::send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks) { + for (size_t l = 0; l < blocks.size(); l += MAX_BLOCKS_PER_MESSAGE) { + size_t r = std::min(l + MAX_BLOCKS_PER_MESSAGE, blocks.size()); + auto query = create_tl_object(); + for (size_t i = l; i < r; ++i) { + query->blocks_.push_back(create_tl_block_id(blocks[i])); + } + td::actor::send_closure(rldp_, &rldp2::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); + } +} + +void ShardBlockRetainer::confirm_shard_block_description(td::Ref desc) { + for (const BlockIdExt& block_id : desc->get_chain_blocks()) { + confirm_block(block_id); + } +} + +void ShardBlockRetainer::confirm_block(BlockIdExt block_id) { + if (is_block_outdated(block_id) || !confirmed_blocks_.insert(block_id).second) { + return; + } + size_t sent = 0; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (shard_intersects(it->first.second, block_id.shard_full())) { + ++sent; + send_confirmations(it->first.first, {block_id}); + } + ++it; + } + LOG(INFO) << "Confirmed block " << block_id.to_str() << ", sending " << sent << " confirmations"; +} + +void ShardBlockRetainer::got_block_from_db(BlockIdExt block_id) { + if (!is_block_outdated(block_id)) { + LOG(INFO) << "Loaded confirmed block from DB: " << block_id.to_str(); + confirm_block(block_id); + } +} + +bool ShardBlockRetainer::is_block_outdated(const BlockIdExt& block_id) const { + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +} // namespace ton::validator diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp new file mode 100644 index 000000000..366a6d506 --- /dev/null +++ b/validator/shard-block-retainer.hpp @@ -0,0 +1,73 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp2/rldp.h" +#include + +namespace ton::validator { + +class ShardBlockRetainer : public td::actor::Actor { + public: + ShardBlockRetainer(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void new_shard_block_description(td::Ref desc); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + bool inited_ = false; + std::set validator_adnl_ids_; + std::map, td::Timestamp> subscribers_; + std::set confirmed_blocks_; + + void process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks); + + void confirm_shard_block_description(td::Ref desc); + void confirm_block(BlockIdExt block_id); + + void got_block_from_db(BlockIdExt block_id); + + bool is_block_outdated(const BlockIdExt& block_id) const; + + static constexpr double SUBSCRIPTION_TTL = 60.0; + static constexpr size_t MAX_BLOCKS_PER_MESSAGE = 8; +}; + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp new file mode 100644 index 000000000..0403ff22a --- /dev/null +++ b/validator/shard-block-verifier.cpp @@ -0,0 +1,196 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-verifier.hpp" + +#include "td/actor/MultiPromise.h" + +namespace ton::validator { + +void ShardBlockVerifier::start_up() { + update_config(opts_->get_shard_block_verifier_config()); + update_masterchain_state(last_masterchain_state_); + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + td::actor::send_closure(id_, &ShardBlockVerifier::process_message, src, std::move(data)); + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); +} + +void ShardBlockVerifier::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID)); +} + +void ShardBlockVerifier::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = std::move(state); + for (auto it = blocks_.begin(); it != blocks_.end();) { + if (is_block_outdated(it->first)) { + it->second.finalize_promises(); + it = blocks_.erase(it); + } else { + ++it; + } + } +} + +void ShardBlockVerifier::wait_shard_blocks(std::vector blocks, td::Promise promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + for (const BlockIdExt& block_id : blocks) { + BlockInfo* info = get_block_info(block_id); + if (info && !info->confirmed) { + info->promises.push_back(ig.get_promise()); + } + } +} + +void ShardBlockVerifier::update_config(td::Ref new_config) { + auto old_config = std::move(config_); + config_ = std::move(new_config); + auto old_blocks = std::move(blocks_); + blocks_.clear(); + for (auto& [block_id, old_info] : old_blocks) { + BlockInfo* new_info = get_block_info(block_id); + if (new_info == nullptr) { + old_info.finalize_promises(); + continue; + } + new_info->promises = std::move(old_info.promises); + if (new_info->confirmed) { + new_info->finalize_promises(); + } + for (size_t old_src_idx = 0; old_src_idx < old_info.confirmed_by.size(); ++old_src_idx) { + if (old_info.confirmed_by[old_src_idx]) { + set_block_confirmed(old_config->shards[old_info.config_shard_idx].trusted_nodes[old_src_idx], block_id); + } + } + } + + alarm_timestamp().relax(send_subscribe_at_ = td::Timestamp::now()); +} + +void ShardBlockVerifier::alarm() { + if (send_subscribe_at_ && send_subscribe_at_.is_in_past()) { + for (auto& shard_config : config_->shards) { + for (auto& node_id : shard_config.trusted_nodes) { + td::Promise P = [shard = shard_config.shard_id, node_id](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Subscribe to " << node_id << " for " << shard.to_str() << " : " << R.move_as_error(); + } + }; + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), + td::Timestamp::in(3.0), + create_serialize_tl_object( + create_tl_shard_id(shard_config.shard_id), 0)); + } + } + send_subscribe_at_ = td::Timestamp::in(SEND_SUBSCRIBE_PERIOD); + } + alarm_timestamp().relax(send_subscribe_at_); +} + +void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto r_obj = fetch_tl_object(data, true); + if (r_obj.is_error()) { + return; + } + for (const auto& b : r_obj.ok()->blocks_) { + set_block_confirmed(src, create_block_id(b)); + } +} + +int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { + for (size_t i = 0; i < config_->shards.size(); i++) { + if (shard_intersects(shard_id, config_->shards[i].shard_id)) { + return (int)i; + } + } + return -1; +} + +bool ShardBlockVerifier::is_block_outdated(const BlockIdExt& block_id) const { + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +ShardBlockVerifier::BlockInfo* ShardBlockVerifier::get_block_info(const BlockIdExt& block_id) { + auto it = blocks_.find(block_id); + if (it != blocks_.end()) { + return &it->second; + } + int config_shard_idx = get_config_shard_idx(block_id.shard_full()); + if (config_shard_idx < 0 || is_block_outdated(block_id)) { + return nullptr; + } + auto& shard_config = config_->shards[config_shard_idx]; + BlockInfo& info = blocks_[block_id]; + info.config_shard_idx = config_shard_idx; + info.confirmed_by.resize(shard_config.trusted_nodes.size(), false); + info.confirmed = (shard_config.required_confirms == 0); + return &info; +} + +void ShardBlockVerifier::set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id) { + BlockInfo* info = get_block_info(block_id); + if (info == nullptr) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : ignored"; + return; + } + auto& shard_config = config_->shards[info->config_shard_idx]; + size_t src_idx = 0; + while (src_idx < shard_config.trusted_nodes.size() && shard_config.trusted_nodes[src_idx] != src) { + ++src_idx; + } + if (src_idx == shard_config.trusted_nodes.size()) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : unknown src"; + return; + } + if (info->confirmed_by[src_idx]) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : duplicate"; + return; + } + info->confirmed_by[src_idx] = true; + ++info->confirmed_by_cnt; + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : accepted (" + << info->confirmed_by_cnt << "/" << shard_config.required_confirms << "/" + << shard_config.trusted_nodes.size() << ")" + << (info->confirmed_by_cnt == shard_config.required_confirms ? ", CONFIRMED" : ""); + if (info->confirmed_by_cnt == shard_config.required_confirms) { + info->confirmed = true; + info->finalize_promises(); + info->promises.clear(); + } +} + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp new file mode 100644 index 000000000..025996681 --- /dev/null +++ b/validator/shard-block-verifier.hpp @@ -0,0 +1,90 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp2/rldp.h" +#include + +namespace ton::validator { + +class ShardBlockVerifier : public td::actor::Actor { + public: + ShardBlockVerifier(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void wait_shard_blocks(std::vector blocks, td::Promise promise); + + void update_options(td::Ref opts) { + if (config_ != opts->get_shard_block_verifier_config()) { + update_config(opts->get_shard_block_verifier_config()); + } + opts_ = std::move(opts); + } + + void alarm() override; + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + td::Ref config_; + + td::Timestamp send_subscribe_at_ = td::Timestamp::never(); + + struct BlockInfo { + size_t config_shard_idx = 0; + std::vector confirmed_by; + td::uint32 confirmed_by_cnt = 0; + bool confirmed = false; + std::vector> promises; + + void finalize_promises() { + for (auto& promise : promises) { + promise.set_value(td::Unit()); + } + } + }; + std::map blocks_; + + void update_config(td::Ref new_config); + void process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); + + int get_config_shard_idx(const ShardIdFull& shard_id) const; + bool is_block_outdated(const BlockIdExt& block_id) const; + BlockInfo* get_block_info(const BlockIdExt& block_id); + + void set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id); + + static constexpr double SEND_SUBSCRIBE_PERIOD = 10.0; +}; + +} // namespace ton::validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index ac86cf37d..49ee7b6e8 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -74,24 +74,29 @@ void ShardClient::got_init_state_from_db(td::Ref state) { } void ShardClient::start_up_init_mode() { - std::vector shards; - for (const auto& s : masterchain_state_->get_shards()) { + std::vector shards; + for (const auto &s : masterchain_state_->get_shards()) { if (opts_->need_monitor(s->shard(), masterchain_state_)) { - shards.push_back(s->top_block_id()); + auto shard = s->top_block_id(); + shards.push_back({ + .shard = shard, + .split_depth = masterchain_state_->persistent_state_split_depth(shard.shard_full().workchain), + }); } } download_shard_states(masterchain_block_handle_->id(), std::move(shards), 0); } -void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx) { +void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, + size_t idx) { if (idx >= shards.size()) { LOG(WARNING) << "downloaded all shard states"; applied_all_shards(); return; } - BlockIdExt block_id = shards[idx]; + auto [block_id, split_depth] = shards[idx]; td::actor::create_actor( - "downloadstate", block_id, masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600 * 5), + "downloadstate", block_id, masterchain_block_handle_->id(), split_depth, 2, manager_, td::Timestamp::in(3600 * 5), [=, SelfId = actor_id(this), shards = std::move(shards)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ShardClient::download_shard_states, masterchain_block_id, std::move(shards), @@ -156,7 +161,7 @@ void ShardClient::download_masterchain_state() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, masterchain_block_handle_, - shard_client_priority(), td::Timestamp::in(600), std::move(P)); + shard_client_priority(), td::Timestamp::in(600), true, std::move(P)); } void ShardClient::got_masterchain_block_state(td::Ref state) { @@ -196,7 +201,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { @@ -211,7 +216,7 @@ void ShardClient::apply_all_shards() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } } diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 7c2c978c2..c13b625aa 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -27,6 +27,11 @@ namespace validator { class ShardClient : public td::actor::Actor { private: + struct DownloadableShard { + BlockIdExt shard; + td::uint32 split_depth; + }; + td::Ref opts_; BlockHandle masterchain_block_handle_; @@ -64,7 +69,7 @@ class ShardClient : public td::actor::Actor { void start_up() override; void start_up_init_mode(); - void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); + void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); void start(); void got_state_from_db(BlockIdExt masterchain_block_id); void got_init_handle_from_db(BlockHandle handle); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index bc3d7b5e0..b024dc100 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -17,11 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "state-serializer.hpp" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" #include "td/utils/Random.h" +#include "td/utils/overloaded.h" #include "ton/ton-io.hpp" #include "common/delay.h" #include "td/utils/filesystem.h" #include "td/utils/HashSet.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -199,7 +203,7 @@ void AsyncStateSerializer::next_iteration() { } if (next_idx_ < shards_.size()) { running_ = true; - request_shard_state(shards_[next_idx_]); + request_shard_state(shards_[next_idx_].block_id); return; } LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); @@ -247,7 +251,10 @@ void AsyncStateSerializer::store_persistent_state_description(td::Refget_unix_time(); desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); for (const auto &v : state->get_shards()) { - desc.shard_blocks.push_back(v->top_block_id()); + desc.shard_blocks.push_back({ + .block = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, td::Ref(true, std::move(desc))); @@ -282,8 +289,38 @@ class CachedCellDbReader : public vm::CellDbReader { } return parent_->load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + total_reqs_ += hashes.size(); + if (!cache_) { + ++bulk_reqs_; + return parent_->load_bulk(hashes); + } + std::vector missing_hashes; + std::vector missing_indices; + std::vector> res(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto it = cache_->find(hashes[i]); + if (it != cache_->end()) { + ++cached_reqs_; + TRY_RESULT(loaded_cell, (*it)->load_cell()); + res[i] = loaded_cell.data_cell; + continue; + } + missing_hashes.push_back(hashes[i]); + missing_indices.push_back(i); + } + if (missing_hashes.empty()) { + return std::move(res); + } + TRY_RESULT(missing_cells, parent_->load_bulk(missing_hashes)); + for (size_t i = 0; i < missing_indices.size(); i++) { + res[missing_indices[i]] = missing_cells[i]; + } + return res; + }; void print_stats() const { - LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached"; + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached, " + << bulk_reqs_ << " bulk reqs"; } private: std::shared_ptr parent_; @@ -291,9 +328,19 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 total_reqs_ = 0; td::uint64 cached_reqs_ = 0; + td::uint64 bulk_reqs_ = 0; }; -void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard, PersistentStateType type) { + if (type.get_offset() == type.offset()) { + // Header of a split state is small, so not caching it is fine. + return; + } + + if (type.get_offset() == type.offset()) { + shard = {shard.workchain, type.get().effective_shard_id}; + } + std::vector prev_shards; for (const auto& [_, prev_shard] : state_files) { if (shard_intersects(shard, prev_shard)) { @@ -304,7 +351,6 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) return; } cur_shards = std::move(prev_shards); - cache = {}; if (cur_shards.empty()) { return; } @@ -344,6 +390,25 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) cache = std::make_shared(std::move(cells)); } +void AsyncStateSerializer::PreviousStateCache::add_new_cells(vm::CellDbReader& reader, Ref const& cell) { + if (!cell->is_loaded()) { + return; + } + if (reader.load_cell(cell->get_hash().as_slice()).is_ok()) { + return; + } + + auto [_, inserted] = cache->insert(cell); + if (!inserted) { + return; + } + + vm::CellSlice cs{vm::NoVm{}, cell}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + add_new_cells(reader, cs.prefetch_ref(i)); + } +} + void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { if (!opts_->get_state_serializer_enabled() || auto_disabled_) { @@ -358,7 +423,10 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state auto vec = state->get_shards(); for (auto &v : vec) { if (opts_->need_monitor(v->shard(), state)) { - shards_.push_back(v->top_block_id()); + shards_.push_back({ + .block_id = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } } @@ -370,10 +438,10 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); } if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + previous_state_cache->prepare_cache(shard, UnsplitStateType{}); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; @@ -387,7 +455,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), - masterchain_handle_->id(), write_data, std::move(P)); + masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); current_status_ts_ = td::Timestamp::now(); @@ -424,43 +492,137 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } +namespace { + +// Expects `ShardStateUnsplit` as `shard_state_cell`. +std::vector split_shard_state(ShardId shard_id, td::Ref shard_state_cell, int split_depth) { + CHECK(split_depth <= 63); + int shard_prefix_length = shard_pfx_len(shard_id); + if (shard_prefix_length >= static_cast(split_depth)) { + return {{UnsplitStateType{}, std::move(shard_state_cell)}}; + } + + block::gen::ShardStateUnsplit::Record unsplit_shard_state; + bool rc = tlb::unpack_cell(shard_state_cell, unsplit_shard_state); + CHECK(rc); + + std::vector result; + + auto unwrapped_accounts_root = unsplit_shard_state.accounts; + auto accounts_cut = std::make_shared(); + auto accounts_root = vm::UsageCell::create(unwrapped_accounts_root, accounts_cut->root_ptr()); + + // NOTE: Ref constructor expects caller to unwrap HashMapAugE. + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(accounts_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + // Build account dict parts + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + result.push_back({SplitAccountStateType{effective_shard}, account_dict_part.get_wrapped_dict_root()}); + } + } + + auto accounts_proof = vm::MerkleProof::generate_raw(unwrapped_accounts_root, accounts_cut.get()); + + // Build header + unsplit_shard_state.accounts = accounts_proof; + vm::CellBuilder unsplit_shard_state_cb; + rc = tlb::pack(unsplit_shard_state_cb, unsplit_shard_state); + CHECK(rc); + + auto header = unsplit_shard_state_cb.finalize(); + CHECK(header->get_level() <= 1 && header->get_hash(0) == shard_state_cell->get_hash()); + + result.push_back({SplitPersistentStateType{}, vm::CellBuilder::create_merkle_proof(header)}); + + return result; +} + +} // namespace + void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + int archive_split_depth = shards_[next_idx_].split_depth; next_idx_++; + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, - previous_state_cache = previous_state_cache_, - fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), + + auto parts = split_shard_state(state->get_shard().shard, state->root_cell(), archive_split_depth); + CHECK(!parts.empty()); + + write_shard_state(handle, state->get_shard(), cell_db_reader, + std::make_shared>(std::move(parts)), 0); + + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); +} + +void AsyncStateSerializer::write_shard_state(BlockHandle handle, ShardIdFull shard, + std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx) { + auto part = parts->at(idx); + auto type = part.type; + auto cell = part.cell; + + auto write_data = [=, this, cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + CHECK(running_); + + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; if (!cell_db_reader) { - return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + return vm::std_boc_serialize_to_file(cell, fd, 31, std::move(cancellation_token)); } - if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + if (opts_->get_fast_state_serializer_enabled()) { + previous_state_cache_->prepare_cache(shard, type); } - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + if (!previous_state_cache_->cache) { + previous_state_cache_->cache = std::make_shared(); + } + previous_state_cache_->add_new_cells(*cell_db_reader, cell); + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_->cache); + auto res = + vm::boc_serialize_to_file_large(new_cell_db_reader, cell->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error() && R.error().code() == cancelled) { LOG(ERROR) << "Persistent state serialization cancelled"; + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); + return; + } + + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; + if (idx + 1 == parts->size()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); } else { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::write_shard_state, handle, shard, cell_db_reader, parts, + idx + 1); } - td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), - masterchain_handle_->id(), write_data, std::move(P)); - current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " - << state->get_block_id().id.to_str(); - current_status_ts_ = td::Timestamp::now(); + masterchain_handle_->id(), type, write_data, std::move(P)); } void AsyncStateSerializer::fail_handler(td::Status reason) { diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 406ac350a..baa9268db 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -27,6 +27,11 @@ namespace ton { namespace validator { +struct SerializablePart { + PersistentStateType type; + td::Ref cell; +}; + class AsyncStateSerializer : public td::actor::Actor { private: td::uint32 attempt_ = 0; @@ -52,13 +57,18 @@ class AsyncStateSerializer : public td::actor::Actor { bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; - std::vector shards_; + struct ShardSerializationConfig { + BlockIdExt block_id; + td::uint32 split_depth; + }; + std::vector shards_; struct PreviousStateCache { std::vector> state_files; std::shared_ptr cache; std::vector cur_shards; - void prepare_cache(ShardIdFull shard); + void prepare_cache(ShardIdFull shard, PersistentStateType type); + void add_new_cells(vm::CellDbReader& reader, Ref const& cell); }; std::shared_ptr previous_state_cache_; @@ -93,6 +103,8 @@ class AsyncStateSerializer : public td::actor::Actor { void stored_masterchain_state(); void got_shard_handle(BlockHandle handle); void got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader); + void write_shard_state(BlockHandle handle, ShardIdFull shard, std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx); void get_masterchain_seqno(td::Promise promise) { promise.set_result(last_block_id_.id.seqno); diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp new file mode 100644 index 000000000..a81d958f5 --- /dev/null +++ b/validator/storage-stat-cache.cpp @@ -0,0 +1,39 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "storage-stat-cache.hpp" + +namespace ton::validator { + +void StorageStatCache::get_cache(td::Promise(const td::Bits256&)>> promise) { + LOG(DEBUG) << "StorageStatCache::get_cache"; + promise.set_value( + [cache = cache_](const td::Bits256& hash) mutable -> td::Ref { return cache.lookup_ref(hash); }); +} + +void StorageStatCache::update(std::vector, td::uint32>> data) { + for (auto &[cell, size] : data) { + if (size < MIN_ACCOUNT_CELLS) { + continue; + } + td::Bits256 hash = cell->get_hash().bits(); + LOG(DEBUG) << "StorageStatCache::update " << hash.to_hex() << " " << size; + cache_.set_ref(hash, cell); + lru_.put(hash, Deleter{hash, &cache_}, true, size); + } +} + +} // namespace ton::validator diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp new file mode 100644 index 000000000..194a0b79a --- /dev/null +++ b/validator/storage-stat-cache.hpp @@ -0,0 +1,68 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "td/utils/ConcurrentHashTable.h" +#include "td/utils/LRUCache.h" + +#include + +namespace ton::validator { + +class StorageStatCache : public td::actor::Actor { + public: + void get_cache(td::Promise(const td::Bits256&)>> promise); + + // (storage dict root, account total cells) + void update(std::vector, td::uint32>> data); + + private: + vm::Dictionary cache_{256}; + + struct Deleter { + Deleter(const td::Bits256& hash, vm::Dictionary* cache) : hash(hash), cache(cache) { + } + Deleter(const Deleter&) = delete; + Deleter(Deleter&& other) noexcept : hash(other.hash), cache(other.cache) { + other.cache = nullptr; + } + Deleter& operator=(const Deleter&) = delete; + Deleter& operator=(Deleter&& other) noexcept { + hash = other.hash; + cache = other.cache; + other.cache = nullptr; + return *this; + } + ~Deleter() { + if (cache) { + CHECK(cache->lookup_delete_ref(hash).not_null()); + LOG(DEBUG) << "StorageStatCache remove " << hash.to_hex(); + } + } + + td::Bits256 hash = td::Bits256::zero(); + vm::Dictionary* cache; + }; + td::LRUCache lru_{MAX_CACHE_TOTAL_CELLS}; + + static constexpr td::uint64 MAX_CACHE_TOTAL_CELLS = 1 << 24; + + public: + static constexpr td::uint64 MIN_ACCOUNT_CELLS = 4000; +}; + +} // namespace ton::validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 110ccd813..c79dbda4a 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -23,52 +23,87 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/lite-tl.hpp" +#include "td/utils/Random.h" +#include "collator-node/collator-node.hpp" namespace ton { namespace validator { static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInfo &source_info, bool is_masterchain) { - return source_info.first_block_round == source_info.round && source_info.source_priority == 0 && !is_masterchain; + return source_info.priority.first_block_round == source_info.priority.round && source_info.priority.priority == 0 && + !is_masterchain; } -void ValidatorGroup::generate_block_candidate( - validatorsession::BlockSourceInfo source_info, - td::Promise promise) { - td::uint32 round_id = source_info.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; +void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info, + td::Promise promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; } + td::uint32 round_id = source_info.priority.round; + update_round_id(round_id); if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "cannot collate block: group not started")); return; } if (cached_collated_block_) { if (cached_collated_block_->result) { - promise.set_value({cached_collated_block_->result.value().clone(), true}); + auto res = cached_collated_block_->result.value().clone(); + res.is_cached = true; + promise.set_value(std::move(res)); } else { - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + cached_collated_block_->promises.push_back(promise.wrap([](GeneratedCandidate &&res) { + res.is_cached = true; + return std::move(res); })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; - })); - run_collate_query( - shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - validator_set_, opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), - [SelfId = actor_id(this), cache = cached_collated_block_, source_info](td::Result R) mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(source_info), - std::move(cache), std::move(R)); - }, cancellation_token_source_.get_cancellation_token(), /* mode = */ 0); + cached_collated_block_->promises.push_back(std::move(promise)); + td::Promise P = [SelfId = actor_id(this), cache = cached_collated_block_, + source_info](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), + std::move(R)); + }; + + if (optimistic_generation_ && prev_block_ids_.size() == 1 && optimistic_generation_->prev == prev_block_ids_[0] && + optimistic_generation_->round == round_id) { + if (optimistic_generation_->result) { + P.set_value(optimistic_generation_->result.value().clone()); + } else { + optimistic_generation_->promises.push_back( + [=, SelfId = actor_id(this), P = std::move(P), + cancellation_token = + cancellation_token_source_.get_cancellation_token()](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidatorGroup::generate_block_candidate_cont, source_info, std::move(P), + std::move(cancellation_token)); + } else { + P.set_value(R.move_as_ok()); + } + }); + } + return; + } + generate_block_candidate_cont(source_info, std::move(P), cancellation_token_source_.get_cancellation_token()); +} + +void ValidatorGroup::generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, + td::CancellationToken cancellation_token) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, + prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + source_info.priority, validator_set_, max_answer_size, std::move(cancellation_token), + std::move(promise), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, - td::Result R) { + td::Result R) { if (R.is_error()) { for (auto &p : cache->promises) { p.set_error(R.error().clone()); @@ -77,12 +112,15 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo cached_collated_block_ = nullptr; } } else { - auto candidate = R.move_as_ok(); - add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash); + auto c = R.move_as_ok(); + add_available_block_candidate(c.candidate.pubkey.as_bits256(), c.candidate.id, c.candidate.collated_file_hash); if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { - send_block_candidate_broadcast(candidate.id, candidate.data.clone()); + send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); + } + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; } - cache->result = std::move(candidate); + cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); } @@ -91,18 +129,54 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo } void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise) { - td::uint32 round_id = source_info.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; + td::Promise> promise, + td::optional optimistic_prev_block) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } + bool is_optimistic = (bool)optimistic_prev_block; + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic validation in masterchain")); + return; + } + td::uint32 round_id = source_info.priority.round; + if (!is_optimistic) { + update_round_id(round_id); } if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + if (is_optimistic && round_id > last_known_round_id_ + 1) { + promise.set_error(td::Status::Error(ErrorCode::notready, "too new")); + return; + } + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("optimistic validation in masterchain is not supported")); + return; + } auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); block.id = next_block_id; + auto prev = prev_block_ids_; + if (is_optimistic) { + if (round_id > last_known_round_id_) { + ++block.id.id.seqno; + } + optimistic_prev_block.value().id.id = block.id.id; + --optimistic_prev_block.value().id.id.seqno; + if (round_id == last_known_round_id_) { + if (prev_block_ids_ != std::vector{optimistic_prev_block.value().id}) { + promise.set_error(td::Status::Error("wrong prev block for optimistic validation")); + return; + } + optimistic_prev_block = {}; + is_optimistic = false; + } else { + prev = {optimistic_prev_block.value().id}; + } + } CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); @@ -111,47 +185,70 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(), manager = manager_, - validator_set = validator_set_, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - LOG(ERROR) << "failed to validate candidate: " << S; - } - delay_action( - [SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(block), std::move(promise)); - }, - td::Timestamp::in(0.1)); - } else { - auto v = R.move_as_ok(); - v.visit(td::overloaded( - [&](UnixTime ts) { - td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); - td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), - block.id, block.collated_file_hash); - if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, - block.data.clone()); - } - promise.set_value({ts, false}); - }, - [&](CandidateReject reject) { - promise.set_error( - td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); - })); - } - }); + auto it2 = block_collator_node_id_.find(block.id); + adnl::AdnlNodeIdShort collator_node_id = + it2 == block_collator_node_id_.end() ? adnl::AdnlNodeIdShort::zero() : it2->second; + + auto P = td::PromiseCreator::lambda( + [=, SelfId = actor_id(this), block = block.clone(), + optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, + promise = std::move(promise), + collation_manager = collation_manager_](td::Result R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + LOG(ERROR) << "failed to validate candidate: " << S; + } + delay_action( + [SelfId, source_info, block = std::move(block), promise = std::move(promise), + optimistic_prev_block = std::move(optimistic_prev_block)]() mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(block), std::move(promise), std::move(optimistic_prev_block)); + }, + td::Timestamp::in(0.1)); + } else { + auto v = R.move_as_ok(); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); + td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, + block.pubkey.as_bits256(), block.id, block.collated_file_hash); + if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, + block.data.clone()); + } + promise.set_value({ts, false}); + }, + [&](CandidateReject reject) { + if (!collator_node_id.is_zero()) { + td::actor::send_closure(collation_manager, &CollationManager::ban_collator, collator_node_id, + PSTRING() << "bad candidate " << block.id.to_str() << " : " << reject.reason); + } + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); + } + }); if (!started_) { P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - block.id = next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, - td::Timestamp::in(15.0), std::move(P)); + td::Ref optimistic_prev_block_data; + if (is_optimistic) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN( + P, optimistic_prev_block_data, + create_block(optimistic_prev_block.value().id, std::move(optimistic_prev_block.value().data)), + "failed to parse optimistic prev block: "); + } + run_validate_query(std::move(block), + ValidateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev), + .validator_set = validator_set_, + .local_validator_id = local_id_, + .optimistic_prev_block = optimistic_prev_block_data}, + manager_, td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -165,10 +262,8 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so validatorsession::ValidatorSessionStats stats, td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); - td::uint32 round_id = source_info.round; - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + td::uint32 round_id = source_info.priority.round; + update_round_id(round_id + 1); auto sig_set = create_signature_set(std::move(signatures)); validator_set_->check_signatures(root_hash, file_hash, sig_set).ensure(); auto approve_sig_set = create_signature_set(std::move(approve_signatures)); @@ -180,11 +275,13 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so return; } auto next_block_id = create_next_block_id(root_hash, file_hash); - LOG(WARNING) << "Accepted block " << next_block_id; - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); + LOG(WARNING) << "Accepted block " << next_block_id.to_str(); + stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; + // OLD BROADCAST BEHAVIOR: // Creator of the block sends broadcast to public overlays // Creator of the block sends broadcast to private block overlay unless candidate broadcast was sent // Any node sends broadcast to custom overlays unless candidate broadcast was sent @@ -199,8 +296,37 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so if (!sent_candidate) { send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; } + // NEW BROADCAST BEHAVIOR (activate later): + // Masterchain block are broadcasted as Block Broadcast (with signatures). Shard blocks are broadcasted as Block Candidate Broadcast (only block data). + // Public and private overlays: creator sends masterchain blocks, all validators send shard blocks. + // Custom overlays: all nodes send all blocks. + // If the block was broadcasted earlier as a candidate (to private and custom overlays), the broadcast is not repeated. + /*int send_broadcast_mode = 0; + bool sent_candidate = sent_candidate_broadcasts_.contains(next_block_id); + if (!shard_.is_masterchain() || source_info.source.compute_short_id() == local_id_) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + } + } + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; + }*/ + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(sig_set), + std::move(approve_sig_set), send_broadcast_mode, std::move(promise)); + prev_block_ids_ = std::vector{next_block_id}; + cached_collated_block_ = nullptr; + cancellation_token_source_.cancel(); + if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && + optimistic_generation_->prev != next_block_id) { + optimistic_generation_ = {}; + } +} - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_, +void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sig_set, td::Ref approve_sig_set, + int send_broadcast_mode, td::Promise promise, bool is_retry) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { if (R.error().code() == ErrorCode::cancelled) { @@ -208,47 +334,20 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so return; } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, - std::move(promise)); + td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), + std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, std::move(promise), + true); } else { promise.set_value(R.move_as_ok()); } }); - run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast_mode, manager_, - std::move(P)); - prev_block_ids_ = std::vector{next_block_id}; - cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); - cancellation_token_source_.cancel(); -} - -void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref block, - std::vector prev, td::Ref sig_set, - td::Ref approve_sig_set, int send_broadcast_mode, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, - std::move(promise)); - } else { - promise.set_value(R.move_as_ok()); - } - }); - - run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast_mode, manager_, std::move(P)); + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast_mode, monitoring_shard_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); } void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, @@ -259,14 +358,79 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } -BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { - BlockSeqno seqno = 0; - for (auto &p : prev_block_ids_) { - if (seqno < p.id.seqno) { - seqno = p.id.seqno; +void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } + if (shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic generation in masterchain")); + return; + } + if (last_known_round_id_ + 1 != source_info.priority.round) { + promise.set_error(td::Status::Error("too old round")); + return; + } + if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + promise.set_error(td::Status::Error("optimistic generation already in progress")); + return; + } + BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_ = std::make_unique(); + optimistic_generation_->round = source_info.priority.round; + optimistic_generation_->prev = BlockIdExt{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_->promises.push_back(std::move(promise)); + + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_optimistic, source_info, std::move(R)); + }; + LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, min_masterchain_block_id_, + block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + source_info.priority, validator_set_, max_answer_size, + optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), + config_.proto_version); +} + +void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::Result R) { + if (!optimistic_generation_ || optimistic_generation_->round != source_info.priority.round) { + return; + } + if (R.is_error()) { + LOG(WARNING) << "Optimistic generation failed: " << R.move_as_error(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_error(R.error().clone()); } + optimistic_generation_ = {}; + return; } - return BlockIdExt{shard_.workchain, shard_.shard, seqno + 1, root_hash, file_hash}; + GeneratedCandidate c = R.move_as_ok(); + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } + optimistic_generation_->result = std::move(c); + for (auto &promise : optimistic_generation_->promises) { + promise.set_result(optimistic_generation_->result.value().clone()); + } + optimistic_generation_->promises.clear(); +} + +void ValidatorGroup::update_round_id(td::uint32 round) { + if (last_known_round_id_ >= round) { + return; + } + last_known_round_id_ = round; + if (optimistic_generation_ && optimistic_generation_->round < round) { + optimistic_generation_ = {}; + } +} + +BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { + return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } BlockId ValidatorGroup::create_next_block_id_simple() const { @@ -306,14 +470,15 @@ std::unique_ptr ValidatorGroup::ma sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(candidate), std::move(P)); + std::move(candidate), std::move(P), td::optional{}); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, - td::Promise promise) override { + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), std::move(promise)); } - void on_block_committed(validatorsession::BlockSourceInfo source_info, validatorsession::ValidatorSessionRootHash root_hash, + void on_block_committed(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, std::vector> approve_signatures, @@ -341,6 +506,29 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::get_approved_candidate, source, root_hash, file_hash, collated_data_file_hash, std::move(promise)); } + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) override { + td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), + prev_root_hash, prev_file_hash, std::move(promise)); + } + void on_optimistic_candidate(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, PublicKey prev_source, + validatorsession::ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) override { + BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, + sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; + BlockCandidate prev_candidate{Ed25519_PublicKey{prev_source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, prev_root_hash, sha256_bits256(prev_data.as_slice())}, + sha256_bits256(prev_collated_data.as_slice()), prev_data.clone(), + prev_collated_data.clone()}; + + td::actor::send_closure( + id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), std::move(candidate), + [](td::Result>) mutable {}, std::move(prev_candidate)); + } private: td::actor::ActorId id_; @@ -359,28 +547,32 @@ void ValidatorGroup::create_session() { validatorsession::ValidatorSessionNode n; n.pub_key = ValidatorFullId{el.key}; n.weight = el.weight; - if (n.pub_key.compute_short_id() == local_id_) { - CHECK(!found); - found = true; - local_id_full_ = n.pub_key; - } if (el.addr.is_zero()) { n.adnl_id = adnl::AdnlNodeIdShort{n.pub_key.compute_short_id()}; } else { n.adnl_id = adnl::AdnlNodeIdShort{el.addr}; } + if (n.pub_key.compute_short_id() == local_id_) { + CHECK(!found); + found = true; + local_id_full_ = n.pub_key; + local_adnl_id_ = n.adnl_id; + } vec.emplace_back(std::move(n)); } CHECK(found); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), - make_validator_session_callback(), keyring_, adnl_, rldp_, + make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_, "-", allow_unsafe_self_blocks_resync_); } else { session_ = validatorsession::ValidatorSession::create( - session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, + session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_ + "/catchains/", PSTRING() << "." << shard_.workchain << "." << shard_.shard << "." << validator_set_->get_catchain_seqno() << ".", @@ -401,7 +593,6 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); started_ = true; if (init_) { @@ -410,36 +601,63 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch for (auto &p : postponed_accept_) { auto next_block_id = create_next_block_id(p.root_hash, p.file_hash); - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, - std::move(p.stats)); + p.stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(p.stats)); auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; - retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs), - std::move(p.approve_sigs), 0, std::move(p.promise)); + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(p.sigs), + std::move(p.approve_sigs), 0, std::move(p.promise)); prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); - validatorsession::NewValidatorGroupStats stats; - stats.session_id = session_id_; - stats.shard = shard_; - stats.cc_seqno = validator_set_->get_catchain_seqno(); - stats.last_key_block_seqno = last_key_block_seqno_; - stats.timestamp = td::Clocks::system(); + validatorsession::NewValidatorGroupStats stats{.session_id = session_id_, + .shard = shard_, + .cc_seqno = validator_set_->get_catchain_seqno(), + .last_key_block_seqno = last_key_block_seqno_, + .started_at = td::Clocks::system(), + .prev = prev, + .self = local_id_}; td::uint32 idx = 0; - for (const auto& node : validator_set_->export_vector()) { + for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); if (id == local_id_) { stats.self_idx = idx; } - stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{ + .id = id, + .pubkey = PublicKey(pubkeys::Ed25519(node.key)), + .adnl_id = (node.addr.is_zero() ? adnl::AdnlNodeIdShort{id} : adnl::AdnlNodeIdShort{node.addr}), + .weight = node.weight}); ++idx; } td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); } void ValidatorGroup::destroy() { + if (destroying_) { + return; + } + destroying_ = true; + if (!session_.empty()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); + } + cancellation_token_source_.cancel(); + delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ValidatorGroup::destroy_cont); }, + td::Timestamp::in(10.0)); +} + +void ValidatorGroup::destroy_cont() { if (!session_.empty()) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_current_stats, [manager = manager_, cc_seqno = validator_set_->get_catchain_seqno(), @@ -454,24 +672,13 @@ void ValidatorGroup::destroy() { return; } stats.cc_seqno = cc_seqno; - td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, - std::move(stats)); - }); - td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, - [manager = manager_](td::Result R) { - if (R.is_error()) { - LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); - return; - } - auto stats = R.move_as_ok(); - td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + stats.block_id = block_id; + td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, std::move(stats)); }); auto ses = session_.release(); - delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, - td::Timestamp::in(10.0)); + td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); } - cancellation_token_source_.cancel(); stop(); } @@ -503,12 +710,12 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( BlockIdExt id{next_block_id, candidate->id_->block_id_->root_hash_, candidate->id_->block_id_->file_hash_}; candidate->id_->block_id_ = create_tl_lite_block_id(id); candidate->available_ = - available_block_candidates_.count({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); + available_block_candidates_.contains({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); } auto result = create_tl_object(); result->next_block_id_ = create_tl_lite_block_id_simple(next_block_id); - for (const BlockIdExt& prev : prev_block_ids_) { + for (const BlockIdExt &prev : prev_block_ids_) { result->prev_.push_back(create_tl_lite_block_id(prev)); } result->cc_seqno_ = validator_set_->get_catchain_seqno(); @@ -516,6 +723,15 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } +void ValidatorGroup::send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { + if (sent_candidate_broadcasts_.insert(id).second) { + td::actor::send_closure( + manager_, &ValidatorManager::send_block_candidate_broadcast, id, validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), std::move(data), + fullnode::FullNode::broadcast_mode_private_block | fullnode::FullNode::broadcast_mode_custom); + } +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index db55614f4..3df456efa 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -18,11 +18,13 @@ */ #pragma once +#include "collation-manager.hpp" #include "interfaces/validator-manager.h" #include "validator-session/validator-session.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include @@ -34,23 +36,29 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(validatorsession::BlockSourceInfo source_info, - td::Promise promise); + void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); + void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise); + td::Promise> promise, + td::optional optimistic_prev_block); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, validatorsession::ValidatorSessionStats stats, td::Promise promise); void skip_round(td::uint32 round); - void retry_accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, - td::Ref sigs, td::Ref approve_sigs, - int send_broadcast_mode, td::Promise promise); + void accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sigs, td::Ref approve_sigs, + int send_broadcast_mode, td::Promise promise, bool is_retry = false); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; BlockId create_next_block_id_simple() const; + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise); + void generated_block_optimistic(validatorsession::BlockSourceInfo source_info, td::Result R); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); @@ -59,22 +67,28 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_started, shard_); + } + void tear_down() override { + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_finished, shard_); } void get_validator_group_info_for_litequery( td::Promise> promise); - void update_options(td::Ref opts) { + void update_options(td::Ref opts, bool apply_blocks) { opts_ = std::move(opts); + monitoring_shard_ = apply_blocks; } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, BlockSeqno last_key_block_seqno, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId overlays, std::string db_root, - td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, td::Ref opts) + td::actor::ActorId rldp2, td::actor::ActorId overlays, + std::string db_root, td::actor::ActorId validator_manager, + td::actor::ActorId collation_manager, bool create_session, + bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -84,16 +98,20 @@ class ValidatorGroup : public td::actor::Actor { , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , overlays_(overlays) , db_root_(std::move(db_root)) , manager_(validator_manager) + , collation_manager_(collation_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) - , opts_(std::move(opts)) { + , opts_(std::move(opts)) + , monitoring_shard_(monitoring_shard) { } private: std::unique_ptr make_validator_session_callback(); + void destroy_cont(); struct PostponedAccept { RootHash root_hash; @@ -122,26 +140,33 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; std::string db_root_; td::actor::ActorId manager_; + td::actor::ActorId collation_manager_; td::actor::ActorOwn session_; + adnl::AdnlNodeIdShort local_adnl_id_; bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; td::Ref opts_; td::uint32 last_known_round_id_ = 0; + bool monitoring_shard_ = true; + bool destroying_ = false; struct CachedCollatedBlock { - td::optional result; - std::vector> promises; + td::optional result; + std::vector> promises; }; std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; + void update_round_id(td::uint32 round); + void generated_block_candidate(validatorsession::BlockSourceInfo source_info, - std::shared_ptr cache, td::Result R); + std::shared_ptr cache, td::Result R); using CacheKey = std::tuple; std::map approved_candidates_cache_; @@ -153,8 +178,7 @@ class ValidatorGroup : public td::actor::Actor { } void get_validator_group_info_for_litequery_cont( - td::uint32 expected_round, - std::vector> candidates, + td::uint32 expected_round, std::vector> candidates, td::Promise> promise); std::set> available_block_candidates_; // source, id, collated hash @@ -164,14 +188,24 @@ class ValidatorGroup : public td::actor::Actor { } std::set sent_candidate_broadcasts_; + std::map block_collator_node_id_; + + void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); - void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { - if (sent_candidate_broadcasts_.insert(id).second) { - td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id, - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - std::move(data)); + struct OptimisticGeneration { + td::uint32 round = 0; + BlockIdExt prev; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + ~OptimisticGeneration() { + for (auto& promise : promises) { + promise.set_error(td::Status::Error(ErrorCode::cancelled, "Cancelled")); + } } - } + }; + std::unique_ptr optimistic_generation_; }; } // namespace validator diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d..a4a61373c 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -20,18 +20,89 @@ #include "ton/ton-shard.h" +#include + namespace ton { namespace validator { -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { - return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), - allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, +td::Status CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { + shards.clear(); + self_collate = false; + for (const auto& shard_obj : obj.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (shard_id.is_masterchain()) { + return td::Status::Error("masterchain shard in collators list"); + } + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard_id.to_str()); + } + shards.emplace_back(); + Shard& shard = shards.back(); + shard.shard_id = shard_id; + shard.self_collate = shard_obj->self_collate_; + if (shard.self_collate) { + self_collate = true; + } + if (shard_obj->select_mode_.empty() || shard_obj->select_mode_ == "random") { + shard.select_mode = mode_random; + } else if (shard_obj->select_mode_ == "ordered") { + shard.select_mode = mode_ordered; + } else if (shard_obj->select_mode_ == "round_robin") { + shard.select_mode = mode_round_robin; + } else { + return td::Status::Error(PSTRING() << "invalid select mode '" << shard_obj->select_mode_ + << "' (allowed: 'random', 'ordered', 'round_robin')"); + } + for (const auto& collator : shard_obj->collators_) { + shard.collators.push_back(adnl::AdnlNodeIdShort{collator->adnl_id_}); + } + } + return td::Status::OK(); +} + +CollatorsList CollatorsList::default_list() { + CollatorsList list; + list.shards.push_back( + {.shard_id = ShardIdFull{basechainId, shardIdAll}, .select_mode = mode_random, .self_collate = true}); + list.self_collate = true; + return list; +} + +td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj) { + shards.clear(); + for (const auto& shard_obj : obj.shards_) { + Shard shard; + shard.shard_id = create_shard_id(shard_obj->shard_id_); + if (shard.shard_id.is_masterchain() || !shard.shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.shard_id.to_str()); + } + std::set trusted_nodes; + for (const td::Bits256& id : shard_obj->trusted_nodes_) { + adnl::AdnlNodeIdShort node_id{id}; + if (!trusted_nodes.insert(node_id).second) { + return td::Status::Error(PSTRING() << "duplicate node " << node_id); + } + shard.trusted_nodes.push_back(node_id); + } + if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { + return td::Status::Error(PSTRING() + << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " + << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + } + shard.required_confirms = shard_obj->required_confirms_; + shards.push_back(std::move(shard)); + } + return td::Status::OK(); +} + +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { + return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, + sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ace6b1066..8c7401625 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -34,7 +34,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split), + state->get_seqno()); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -139,6 +140,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_in_memory() const override { return celldb_in_memory_; } + bool get_celldb_v2() const override { + return celldb_v2_; + } + bool get_celldb_disable_bloom_filter() const override { + return celldb_disable_bloom_filter_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -157,6 +164,18 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double get_catchain_broadcast_speed_multiplier() const override { return catchain_broadcast_speed_multipliers_; } + bool get_permanent_celldb() const override { + return permanent_celldb_; + } + td::Ref get_collators_list() const override { + return collators_list_; + } + bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const override { + return !collator_node_whitelist_enabled_ || collator_node_whitelist_.contains(id); + } + td::Ref get_shard_block_verifier_config() const override { + return shard_block_verifier_config_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -164,7 +183,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -237,6 +256,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_in_memory(bool value) override { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) override { + celldb_v2_ = value; + } + void set_celldb_disable_bloom_filter(bool value) override { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -255,18 +280,35 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_broadcast_speed_multiplier(double value) override { catchain_broadcast_speed_multipliers_ = value; } + void set_permanent_celldb(bool value) override { + permanent_celldb_ = value; + } + void set_collators_list(td::Ref list) override { + collators_list_ = std::move(list); + } + void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) override { + if (add) { + collator_node_whitelist_.insert(id); + } else { + collator_node_whitelist_.erase(id); + } + } + void set_collator_node_whitelist_enabled(bool enabled) override { + collator_node_whitelist_enabled_ = enabled; + } + void set_shard_block_verifier_config(td::Ref config) override { + shard_block_verifier_config_ = std::move(config); + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } - ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) - , check_shard_(std::move(check_shard)) , allow_blockchain_init_(allow_blockchain_init) , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) @@ -280,7 +322,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_ = [](ShardIdFull, BlockSeqno) { return true; }; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -304,11 +346,18 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; double catchain_broadcast_speed_multipliers_; + bool permanent_celldb_ = false; + td::Ref collators_list_{true, CollatorsList::default_list()}; + std::set collator_node_whitelist_; + bool collator_node_whitelist_enabled_ = false; + td::Ref shard_block_verifier_config_{true}; }; } // namespace validator diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index 403dd6f9f..b870e8b84 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -51,7 +51,7 @@ void ValidatorTelemetry::start_up() { cpu_cores_ = r_cpu_cores.move_as_ok(); } - LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + LOG(DEBUG) << "Initializing validator telemetry, adnl_id = " << local_id_; alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); } @@ -81,7 +81,7 @@ void ValidatorTelemetry::send_telemetry() { .cpu_threads_count; LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; - td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); + callback_->send_telemetry(std::move(telemetry)); } } // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp index 73908bdd1..d83641dfe 100644 --- a/validator/validator-telemetry.hpp +++ b/validator/validator-telemetry.hpp @@ -33,23 +33,23 @@ namespace ton::validator { class ValidatorManager; class ValidatorTelemetry : public td::actor::Actor { -public: - ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, - td::actor::ActorId manager) - : key_(key) - , local_id_(local_id) - , zero_state_file_hash_(zero_state_file_hash) - , manager_(std::move(manager)) { + public: + class Callback { + public: + virtual ~Callback() = default; + virtual void send_telemetry(tl_object_ptr telemetry) = 0; + }; + + ValidatorTelemetry(adnl::AdnlNodeIdShort local_id, std::unique_ptr callback) + : local_id_(local_id), callback_(std::move(callback)) { } void start_up() override; void alarm() override; -private: - PublicKeyHash key_; + private: adnl::AdnlNodeIdShort local_id_; - td::Bits256 zero_state_file_hash_; - td::actor::ActorId manager_; + std::unique_ptr callback_; std::string node_version_; std::string os_version_; @@ -61,6 +61,5 @@ class ValidatorTelemetry : public td::actor::Actor { void send_telemetry(); static constexpr double PERIOD = 600.0; - static constexpr td::uint32 MAX_SIZE = 8192; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c..826fc8f8f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -22,6 +22,7 @@ #include #include +#include "interfaces/persistent-state.h" #include "td/actor/actor.h" #include "ton/ton-types.h" @@ -71,12 +72,43 @@ struct CollatorOptions : public td::CntObject { std::set> whitelist; // Prioritize these accounts on each phase of process_dispatch_queue std::set> prioritylist; + + // Always enable full_collated_data + bool force_full_collated_data = false; + // Ignore collated data size limits from block limits and catchain config + bool ignore_collated_data_limits = false; +}; + +struct CollatorsList : public td::CntObject { + enum SelectMode { + mode_random, mode_ordered, mode_round_robin + }; + struct Shard { + ShardIdFull shard_id; + SelectMode select_mode = mode_random; + std::vector collators; + bool self_collate = false; + }; + std::vector shards; + bool self_collate = false; + + td::Status unpack(const ton_api::engine_validator_collatorsList& obj); + static CollatorsList default_list(); +}; + +struct ShardBlockVerifierConfig : public td::CntObject { + struct Shard { + ShardIdFull shard_id; + std::vector trusted_nodes; + td::uint32 required_confirms; + }; + std::vector shards; + + td::Status unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj); }; struct ValidatorManagerOptions : public td::CntObject { public: - enum class ShardCheckMode { m_monitor, m_validate }; - virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; @@ -104,6 +136,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; virtual bool get_celldb_in_memory() const = 0; + virtual bool get_celldb_v2() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -111,16 +144,21 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::optional get_celldb_cache_size() const = 0; virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; + virtual bool get_celldb_disable_bloom_filter() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual td::optional get_catchain_max_block_delay_slow() const = 0; virtual bool get_state_serializer_enabled() const = 0; virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual double get_catchain_broadcast_speed_multiplier() const = 0; + virtual bool get_permanent_celldb() const = 0; + virtual td::Ref get_collators_list() const = 0; + virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; + virtual td::Ref get_shard_block_verifier_config() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -144,17 +182,22 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_celldb_v2(bool value) = 0; + virtual void set_celldb_disable_bloom_filter(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; + virtual void set_permanent_celldb(bool value) = 0; + virtual void set_collators_list(td::Ref list) = 0; + virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0; + virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; + virtual void set_shard_block_verifier_config(td::Ref config) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - - std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); @@ -174,14 +217,18 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; + virtual void send_out_msg_queue_proof_broadcast(td::Ref broadcats) { + LOG(ERROR) << "Unimplemented send_out_msg_queue_proof_broadcast - ignore broadcast"; + } virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -195,7 +242,6 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; @@ -212,7 +258,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) = 0; virtual void validate_block(ReceivedBlock block, td::Promise promise) = 0; - virtual void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) = 0; + virtual void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) = 0; //virtual void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; virtual void sync_complete(td::Promise promise) = 0; @@ -227,12 +273,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; - virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise) = 0; + virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; + virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; virtual void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; @@ -248,8 +295,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void new_external_message(td::BufferSlice data, int priority) = 0; virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; - virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; + virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) = 0; + virtual void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; @@ -276,11 +324,15 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; + virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, + td::Timestamp timeout, + td::Promise>> promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -302,6 +354,20 @@ class ValidatorManagerInterface : public td::actor::Actor { } virtual void unregister_stats_provider(td::uint64 idx) { } + + virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + + virtual void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(ERROR) << "Unimplemented add_out_msg_queu_proof - ignore broadcast"; + } + + virtual void get_collation_manager_stats( + td::Promise> promise) = 0; + + virtual void add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + LOG(ERROR) << "Unimplemented add_shard_block_retainer"; + } }; } // namespace validator