Merge developer branch

.github/workflows/build-ton-linux-x86-64-shared.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -21,7 +21,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev
 
-    - name: Install clang-16
+    - if: matrix.os != 'ubuntu-24.04'
       run: |
         wget https://apt.llvm.org/llvm.sh
         chmod +x llvm.sh

.github/workflows/build-ton-wasm-emscripten.yml

@@ -19,7 +19,7 @@ jobs:
 
     - name: Build TON WASM artifacts
       run: |
-        cd assembly/wasm
+        cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
         chmod +x fift-func-wasm-build-ubuntu.sh
         ./fift-func-wasm-build-ubuntu.sh -a
 

.github/workflows/docker-ubuntu-image.yml

@@ -20,22 +20,49 @@ jobs:
           submodules: 'recursive'
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          load: true
+          context: ./
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Test
+        run: |
+          docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Get next tag
+        id: tag
+        run: |
+          git fetch --all --tags
+          git tag -l
+          NEW_TAG=v$(date +'%Y.%m')
+          FOUND=$(git tag -l | grep $NEW_TAG | wc -l)
+          if [ $FOUND -eq 0 ]; then
+            echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT
+          else
+            echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
+          platforms: linux/amd64,linux/arm64
           push: true
           context: ./
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}

.github/workflows/ton-x86-64-windows.yml

@@ -9,7 +9,7 @@ defaults:
 jobs:
   build:
 
-    runs-on: windows-2022
+    runs-on: windows-2019
 
     steps:
       - name: Get Current OS version
@@ -23,9 +23,9 @@ jobs:
 
       - name: Build TON
         run: |
-          copy assembly\native\build-windows-github.bat .
-          copy assembly\native\build-windows.bat .
-          build-windows-github.bat Enterprise
+          copy assembly\native\build-windows-github-2019.bat .
+          copy assembly\native\build-windows-2019.bat .
+          build-windows-github-2019.bat Enterprise
 
       - name: Upload artifacts
         uses: actions/upload-artifact@master

CMake/FindSodium.cmake

@@ -37,12 +37,14 @@ if (NOT DEFINED SODIUM_USE_STATIC_LIBS)
   option(SODIUM_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
 endif()
 if(NOT (SODIUM_USE_STATIC_LIBS EQUAL SODIUM_USE_STATIC_LIBS_LAST))
-  unset(sodium_LIBRARY CACHE)
-  unset(SODIUM_LIBRARY_DEBUG CACHE)
-  unset(SODIUM_LIBRARY_RELEASE CACHE)
-  unset(sodium_DLL_DEBUG CACHE)
-  unset(sodium_DLL_RELEASE CACHE)
-  set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  if (NOT SODIUM_LIBRARY_RELEASE)
+    unset(sodium_LIBRARY CACHE)
+    unset(SODIUM_LIBRARY_DEBUG CACHE)
+    unset(SODIUM_LIBRARY_RELEASE CACHE)
+    unset(sodium_DLL_DEBUG CACHE)
+    unset(sodium_DLL_RELEASE CACHE)
+    set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  endif()
 endif()
 
 
@@ -295,4 +297,4 @@ else()
         )
     endif()
   endif()
-endif()
+endif()

CMakeLists.txt

@@ -539,6 +539,9 @@ target_link_libraries(test-ton-collator overlay tdutils tdactor adnl tl_api dht
 add_executable(test-http test/test-http.cpp)
 target_link_libraries(test-http PRIVATE tonhttp)
 
+add_executable(test-emulator test/test-td-main.cpp emulator/test/emulator-tests.cpp)
+target_link_libraries(test-emulator PRIVATE emulator)
+
 get_directory_property(HAS_PARENT PARENT_DIRECTORY)
 if (HAS_PARENT)
   set(ALL_TEST_SOURCE
@@ -570,6 +573,7 @@ add_test(test-cells test-cells ${TEST_OPTIONS})
 add_test(test-smartcont test-smartcont)
 add_test(test-net test-net)
 add_test(test-actors test-tdactor)
+add_test(test-emulator test-emulator)
 
 #BEGIN tonlib
 add_test(test-tdutils test-tdutils)

Changelog.md

@@ -1,3 +1,20 @@
+## 2024.08 Update
+
+1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. 
+2. A number of changes to transcation executor which will activated for `Config8.version >= 8`:
+    - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set.
+    - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`.
+    - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent.
+    - Allow unfreeze through external messages.
+    - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages.
+3. A few issues with broadcasts were fixed: stop on receiving last piece, response to AdnlMessageCreateChannel
+4. A number of fixes and improvements for emulator and tonlib: correct work with config_addr, not accepted externals, bounces, debug ops gas consumption, added version and c5 dump, fixed tonlib crashes
+5. Added new flags and commands to the node, in particular `--fast-state-serializer`, `getcollatoroptionsjson`, `setcollatoroptionsjson`
+
+Besides the work of the core team, this update is based on the efforts of @krigga (emulator), stonfi team, in particular @dbaranovstonfi and @hey-researcher (emulator), and  @loeul, @xiaoxianBoy, @simlecode (typos in comments and docs).
+
+
+
 ## 2024.06 Update
 
 1. Make Jemalloc default allocator

Dockerfile

@@ -1,10 +1,15 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \
-	rm -rf /var/lib/apt/lists/*
-ENV CC clang
-ENV CXX clang++
-ENV CCACHE_DISABLE 1
+        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh 16 all && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CC=/usr/bin/clang-16
+ENV CXX=/usr/bin/clang++-16
+ENV CCACHE_DISABLE=1
 
 WORKDIR /
 RUN mkdir ton
@@ -13,17 +18,16 @@ WORKDIR /ton
 COPY ./ ./
 
 RUN mkdir build && \
-	cd build && \
-	cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
-	ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
+        cd build && \
+        cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
+        ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \
+    apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /var/ton-work/db && \
-    mkdir -p /var/ton-work/db/static
+RUN mkdir -p /var/ton-work/db /var/ton-work/scripts
 
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
@@ -33,7 +37,7 @@ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console
 COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/
 
 WORKDIR /var/ton-work/db
-COPY ./docker/init.sh ./docker/control.template ./
-RUN chmod +x init.sh
+COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/
+RUN chmod +x /var/ton-work/scripts/init.sh
 
-ENTRYPOINT ["/var/ton-work/db/init.sh"]
+ENTRYPOINT ["/var/ton-work/scripts/init.sh"]

adnl/adnl-ext-server.cpp

@@ -91,7 +91,7 @@ td::Status AdnlInboundConnection::process_custom_packet(td::BufferSlice &data, b
     auto F = fetch_tl_object<ton_api::tcp_authentificate>(data.clone(), true);
     if (F.is_ok()) {
       if (nonce_.size() > 0 || !remote_id_.is_zero()) {
-        return td::Status::Error(ErrorCode::protoviolation, "duplicate authentificate");
+        return td::Status::Error(ErrorCode::protoviolation, "duplicate authenticate");
       }
       auto f = F.move_as_ok();
       nonce_ = td::SecureString{f->nonce_.size() + 256};

adnl/adnl-peer.cpp

@@ -504,6 +504,12 @@ void AdnlPeerPairImpl::create_channel(pubkeys::Ed25519 pub, td::uint32 date) {
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCreateChannel &message) {
   create_channel(message.key(), message.date());
+  if (respond_to_channel_create_after_.is_in_past()) {
+    respond_to_channel_create_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0));
+    std::vector<OutboundAdnlMessage> messages;
+    messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0);
+    send_messages(std::move(messages));
+  }
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChannel &message) {

adnl/adnl-peer.hpp

@@ -214,6 +214,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
   pubkeys::Ed25519 channel_pub_;
   td::int32 channel_pk_date_;
   td::actor::ActorOwn<AdnlChannel> channel_;
+  td::Timestamp respond_to_channel_create_after_;
 
   td::uint64 in_seqno_ = 0;
   td::uint64 out_seqno_ = 0;

assembly/cicd/jenkins/test-builds.groovy

@@ -1,4 +1,5 @@
 pipeline {
+
     agent none
     stages {
         stage('Run Builds') {
@@ -12,7 +13,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
-                                ./build-ubuntu-shared.sh -t -a
+                                ./build-ubuntu-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -31,7 +32,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-x86-64-nix.sh .
                                 chmod +x build-linux-x86-64-nix.sh
-                                ./build-linux-x86-64-nix.sh -t
+                                ./build-linux-x86-64-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -50,7 +51,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
-                                ./build-ubuntu-shared.sh -t -a
+                                ./build-ubuntu-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -69,7 +70,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-arm64-nix.sh .
                                 chmod +x build-linux-arm64-nix.sh
-                                ./build-linux-arm64-nix.sh -t
+                                ./build-linux-arm64-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -88,7 +89,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -107,7 +108,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh -t
+                                ./build-macos-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -126,7 +127,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -145,7 +146,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh -t
+                                ./build-macos-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -164,7 +165,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -182,7 +183,7 @@ pipeline {
                         timeout(time: 180, unit: 'MINUTES') {
                             bat '''
                                 copy assembly\\native\\build-windows.bat .
-                                build-windows.bat -t
+                                build-windows.bat
                             '''
                             bat '''
                                 cd artifacts
@@ -218,7 +219,7 @@ pipeline {
                     steps {
                         timeout(time: 180, unit: 'MINUTES') {
                             sh '''
-                                cd assembly/wasm
+                                cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
                                 chmod +x fift-func-wasm-build-ubuntu.sh
                                 ./fift-func-wasm-build-ubuntu.sh -a
                             '''

assembly/native/build-macos-portable.sh

@@ -158,7 +158,7 @@ if [ "$with_tests" = true ]; then
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
-  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state
+  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \

assembly/native/build-macos-shared.sh

@@ -86,7 +86,7 @@ if [ "$with_tests" = true ]; then
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
-  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state
+  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \

assembly/native/build-ubuntu-portable.sh

@@ -150,7 +150,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
       adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
-      test-fec test-tddb test-db test-validator-session-state
+      test-fec test-tddb test-db test-validator-session-state test-emulator
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
 ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \

assembly/native/build-ubuntu-shared.sh

@@ -58,7 +58,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
       adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
-      test-fec test-tddb test-db test-validator-session-state
+      test-fec test-tddb test-db test-validator-session-state test-emulator
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
 ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \