Merge pull request #5975 from ethereum/develop

Release 0.5.4

.circleci/config.yml

@@ -66,6 +66,7 @@ jobs:
           paths:
             - soljson.js
             - version.txt
+
   test_emscripten_solcjs:
     docker:
       - image: circleci/node:10
@@ -84,6 +85,7 @@ jobs:
           name: Test solcjs
           command: |
             test/solcjsTests.sh /tmp/workspace/soljson.js $(cat /tmp/workspace/version.txt)
+
   test_emscripten_external:
     docker:
       - image: circleci/node:10
@@ -102,6 +104,7 @@ jobs:
           name: External tests
           command: |
             test/externalTests.sh /tmp/workspace/soljson.js || test/externalTests.sh /tmp/workspace/soljson.js
+
   build_x86_linux:
     docker:
       - image: buildpack-deps:bionic
@@ -114,7 +117,7 @@ jobs:
           name: Install build dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libz3-dev
+            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
             ./scripts/install_obsolete_jsoncpp_1_7_4.sh
       - run: *setup_prerelease_commit_hash
       - run: *run_build
@@ -124,6 +127,23 @@ jobs:
           paths:
             - "*"
 
+  build_x86_linux_cxx17:
+    docker:
+      - image: buildpack-deps:disco
+    environment:
+      TERM: xterm
+      CMAKE_OPTIONS: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/cxx17.cmake
+    steps:
+      - checkout
+      - run:
+          name: Install build dependencies
+          command: |
+            apt-get -qq update
+            apt-get -qy install cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
+            ./scripts/install_obsolete_jsoncpp_1_7_4.sh
+      - run: *setup_prerelease_commit_hash
+      - run: *run_build
+
   build_x86_archlinux:
     docker:
       - image: archlinux/base
@@ -145,29 +165,31 @@ jobs:
             - test/soltest
             - test/tools/solfuzzer
 
-  build_x86_clang7:
+  build_x86_clang7_asan:
     docker:
       - image: buildpack-deps:cosmic
     environment:
       TERM: xterm
       CC: /usr/bin/clang-7
       CXX: /usr/bin/clang++-7
-      CMAKE_OPTIONS: -DLLL=ON
+      CMAKE_OPTIONS: -DSANITIZE=address -DCMAKE_BUILD_TYPE=Debug
     steps:
       - checkout
       - run:
           name: Install build dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install clang-7 cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libz3-dev
+            apt-get -qy install clang-7 cmake libboost-regex-dev libboost-filesystem-dev libboost-test-dev libboost-system-dev libboost-program-options-dev libcvc4-dev
             ./scripts/install_obsolete_jsoncpp_1_7_4.sh
       - run: *setup_prerelease_commit_hash
       - run: *run_build
       - store_artifacts: *solc_artifact
       - persist_to_workspace:
           root: build
           paths:
-            - "*"
+            - solc/solc
+            - test/soltest
+            - test/tools/solfuzzer
 
   build_x86_mac:
     macos:
@@ -249,7 +271,7 @@ jobs:
           name: Install dependencies
           command: |
             apt-get -qq update
-            apt-get -qy install libz3-dev libleveldb1v5 python-pip
+            apt-get -qy install libcvc4-dev libleveldb1v5 python-pip
             pip install codecov
       - run: mkdir -p test_results
       - run:
@@ -268,6 +290,36 @@ jobs:
           path: test_results/
           destination: test_results/
 
+  test_x86_clang7_asan:
+    docker:
+      - image: buildpack-deps:cosmic
+    environment:
+      TERM: xterm
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
+      - run:
+          name: Install dependencies
+          command: |
+            apt-get -qq update
+            apt-get -qy install llvm-7-dev libcvc4-dev libleveldb1v5 python-pip
+            # This is needed to resolve the symbols. Since we're using clang7 in the build, we must use the appropriate symbolizer.
+            update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-7 1
+      - run: mkdir -p test_results
+      - run:
+          name: Run tests with ASAN
+          command: |
+            ulimit -a
+            # Increase stack size because ASan makes stack frames bigger and that breaks our assumptions (in tests).
+            ulimit -s 16384
+            build/test/soltest --logger=JUNIT,test_suite,test_results/result.xml -- --no-ipc --testpath test
+      - store_test_results:
+          path: test_results/
+      - store_artifacts:
+          path: test_results/
+          destination: test_results/
+
   test_x86_archlinux:
     docker:
       - image: archlinux/base
@@ -350,12 +402,19 @@ workflows:
           requires:
             - build_emscripten
       - build_x86_linux: *build_on_tags
-      - build_x86_clang7: *build_on_tags
+      - build_x86_linux_cxx17: *build_on_tags
+      - build_x86_clang7_asan: *build_on_tags
       - build_x86_mac: *build_on_tags
       - test_x86_linux:
           <<: *build_on_tags
           requires:
             - build_x86_linux
+      - test_x86_clang7_asan:
+          filters:
+            branches:
+              only: develop
+          requires:
+            - build_x86_clang7_asan
       - test_x86_mac:
           <<: *build_on_tags
           requires:

.travis.yml

@@ -58,13 +58,11 @@ matrix:
           before_install:
               - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
               - sudo add-apt-repository -y ppa:mhier/libboost-latest
-              - sudo add-apt-repository -y ppa:hvr/z3
               - sudo apt-get update -qq
           install:
               - sudo apt-get install -qq g++-8 gcc-8
               - sudo apt-get install -qq libboost1.67-dev
               - sudo apt-get install -qq libleveldb1
-              - sudo apt-get install -qq libz3-dev
               - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90
               - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90
 
@@ -78,13 +76,11 @@ matrix:
           before_install:
               - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
               - sudo add-apt-repository -y ppa:mhier/libboost-latest
-              - sudo add-apt-repository -y ppa:hvr/z3
               - sudo apt-get update -qq
           install:
               - sudo apt-get install -qq g++-8 gcc-8
               - sudo apt-get install -qq libboost1.67-dev
               - sudo apt-get install -qq libleveldb1
-              - sudo apt-get install -qq libz3-dev
               - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90
               - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90
 

CMakeLists.txt

@@ -10,7 +10,7 @@ include(EthPolicy)
 eth_policy()
 
 # project name and version should be set after cmake_policy CMP0048
-set(PROJECT_VERSION "0.5.3")
+set(PROJECT_VERSION "0.5.4")
 project(solidity VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 
 option(LLL "Build LLL" OFF)

Changelog.md

@@ -1,3 +1,32 @@
+### 0.5.4 (2019-02-12)
+
+Language Features:
+ * Allow calldata structs without dynamically encoded members with ABIEncoderV2.
+
+
+Compiler Features:
+ * ABIEncoderV2: Implement packed encoding.
+ * C API (``libsolc`` / raw ``soljson.js``): Introduce ``solidity_free`` method which releases all internal buffers to save memory.
+ * Commandline Interface: Adds new option ``--new-reporter`` for improved diagnostics formatting
+   along with ``--color`` and ``--no-color`` for colorized output to be forced (or explicitly disabled).
+
+
+Bugfixes:
+ * Code Generator: Defensively pad allocation of creationCode and runtimeCode to multiples of 32 bytes.
+ * Commandline Interface: Allow yul optimizer only for strict assembly.
+ * Parser: Disallow empty import statements.
+ * Type Checker: Disallow mappings with data locations other than ``storage``.
+ * Type Checker: Fix internal error when a struct array index does not fit into a uint256.
+ * Type System: Properly report packed encoded size for arrays and structs (mostly unused until now).
+
+
+Build System:
+ * Add support for continuous fuzzing via Google oss-fuzz
+ * SMT: If using Z3, require version 4.6.0 or newer.
+ * Soltest: Add parser that is used in the file-based unit test environment.
+ * Ubuntu PPA Packages: Use CVC4 as SMT solver instead of Z3
+
+
 ### 0.5.3 (2019-01-22)
 
 Language Features:

cmake/EthCompilerSettings.cmake

@@ -74,7 +74,6 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
 			# into errors, which makes sense.
 			# http://stackoverflow.com/questions/21617158/how-to-silence-unused-command-line-argument-error-with-clang-without-disabling-i
 			add_compile_options(-Qunused-arguments)
-
 		elseif(EMSCRIPTEN)
 			set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --memory-init-file 0")
 			# Leave only exported symbols as public and aggressively remove others
@@ -124,6 +123,7 @@ elseif (DEFINED MSVC)
 	add_compile_options(/wd4800)					# disable forcing value to bool 'true' or 'false' (performance warning) (4800)
 	add_compile_options(-D_WIN32_WINNT=0x0600)		# declare Windows Vista API requirement
 	add_compile_options(-DNOMINMAX)					# undefine windows.h MAX && MIN macros cause it cause conflicts with std::min && std::max functions
+	add_compile_options(/utf-8)					# enable utf-8 encoding (solves warning 4819)
 
 	# disable empty object file warning
 	set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /ignore:4221")

cmake/EthOptions.cmake

@@ -3,6 +3,7 @@ macro(configure_project)
 
 	# features
 	eth_default_option(COVERAGE OFF)
+	eth_default_option(OSSFUZZ OFF)
 
 	# components
 	eth_default_option(TESTS ON)

cmake/FindZ3.cmake

@@ -1,8 +1,24 @@
 if (USE_Z3)
     find_path(Z3_INCLUDE_DIR NAMES z3++.h PATH_SUFFIXES z3)
     find_library(Z3_LIBRARY NAMES z3)
+    find_program(Z3_EXECUTABLE z3 PATH_SUFFIXES bin)
+
+    if(Z3_INCLUDE_DIR AND Z3_LIBRARY AND Z3_EXECUTABLE)
+        execute_process (COMMAND ${Z3_EXECUTABLE} -version
+            OUTPUT_VARIABLE libz3_version_str
+            ERROR_QUIET
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+        string(REGEX REPLACE "^Z3 version ([0-9.]+).*" "\\1"
+               Z3_VERSION_STRING "${libz3_version_str}")
+        unset(libz3_version_str)
+    endif()
+    mark_as_advanced(Z3_VERSION_STRING z3_DIR)
+
     include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(Z3 DEFAULT_MSG Z3_LIBRARY Z3_INCLUDE_DIR)
+    find_package_handle_standard_args(Z3
+        REQUIRED_VARS Z3_LIBRARY Z3_INCLUDE_DIR
+        VERSION_VAR Z3_VERSION_STRING)
 
     if (NOT TARGET Z3::Z3)
         add_library(Z3::Z3 UNKNOWN IMPORTED)

cmake/toolchains/cxx17.cmake

@@ -0,0 +1,4 @@
+# Require C++17.
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
+set(CMAKE_CXX_EXTENSIONS OFF)

docs/abi-spec.rst

@@ -418,10 +418,22 @@ In effect, a log entry using this ABI is described as:
 
 - ``address``: the address of the contract (intrinsically provided by Ethereum);
 - ``topics[0]``: ``keccak(EVENT_NAME+"("+EVENT_ARGS.map(canonical_type_of).join(",")+")")`` (``canonical_type_of`` is a function that simply returns the canonical type of a given argument, e.g. for ``uint indexed foo``, it would return ``uint256``). If the event is declared as ``anonymous`` the ``topics[0]`` is not generated;
-- ``topics[n]``: ``EVENT_INDEXED_ARGS[n - 1]`` (``EVENT_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are indexed);
-- ``data``: ``abi_serialise(EVENT_NON_INDEXED_ARGS)`` (``EVENT_NON_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are not indexed, ``abi_serialise`` is the ABI serialisation function used for returning a series of typed values from a function, as described above).
-
-For all fixed-length Solidity types, the ``EVENT_INDEXED_ARGS`` array contains the 32-byte encoded value directly. However, for *types of dynamic length*, which include ``string``, ``bytes``, and arrays, ``EVENT_INDEXED_ARGS`` will contain the *Keccak hash* of the packed encoded value (see :ref:`abi_packed_mode`), rather than the encoded value directly. This allows applications to efficiently query for values of dynamic-length types (by setting the hash of the encoded value as the topic), but leaves applications unable to decode indexed values they have not queried for. For dynamic-length types, application developers face a trade-off between fast search for predetermined values (if the argument is indexed) and legibility of arbitrary values (which requires that the arguments not be indexed). Developers may overcome this tradeoff and achieve both efficient search and arbitrary legibility by defining events with two arguments — one indexed, one not — intended to hold the same value.
+- ``topics[n]``: ``abi_encode(EVENT_INDEXED_ARGS[n - 1])`` (``EVENT_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are indexed);
+- ``data``: ABI encoding of ``EVENT_NON_INDEXED_ARGS`` (``EVENT_NON_INDEXED_ARGS`` is the series of ``EVENT_ARGS`` that are not indexed, ``abi_encode`` is the ABI encoding function used for returning a series of typed values from a function, as described above).
+
+For all types of length at most 32 bytes, the ``EVENT_INDEXED_ARGS`` array contains
+the value directly, padded or sign-extended (for signed integers) to 32 bytes, just as for regular ABI encoding.
+However, for all "complex" types or types of dynamic length, including all arrays, ``string``, ``bytes`` and structs,
+``EVENT_INDEXED_ARGS`` will contain the *Keccak hash* of a special in-place encoded value
+(see :ref:`indexed_event_encoding`), rather than the encoded value directly.
+This allows applications to efficiently query for values of dynamic-length types
+(by setting the hash of the encoded value as the topic), but leaves applications unable
+to decode indexed values they have not queried for. For dynamic-length types,
+application developers face a trade-off between fast search for predetermined values
+(if the argument is indexed) and legibility of arbitrary values (which requires that
+the arguments not be indexed). Developers may overcome this tradeoff and achieve both
+efficient search and arbitrary legibility by defining events with two arguments — one
+indexed, one not — intended to hold the same value.
 
 .. _abi_json:
 
@@ -608,8 +620,9 @@ Through ``abi.encodePacked()``, Solidity supports a non-standard packed mode whe
 
 - types shorter than 32 bytes are neither zero padded nor sign extended and
 - dynamic types are encoded in-place and without the length.
+- array elements are padded, but still encoded in-place
 
-This packed mode is mainly used for indexed event parameters.
+Furthermore, structs as well as nested arrays are not supported.
 
 As an example, the encoding of ``int16(-1), bytes1(0x42), uint16(0x03), string("Hello, world!")`` results in:
 
@@ -622,12 +635,18 @@ As an example, the encoding of ``int16(-1), bytes1(0x42), uint16(0x03), string("
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^ string("Hello, world!") without a length field
 
 More specifically:
- - Each value type takes as many bytes as its range has.
- - The encoding of a struct or fixed-size array is the concatenation of the
-   encoding of its members/elements without any separator or padding.
- - Mapping members of structs are ignored as usual.
- - Dynamically-sized types like ``string``, ``bytes`` or ``uint[]`` are encoded without
-   their length field.
+ - During the encoding, everything is encoded in-place. This means that there is
+   no distinction between head and tail, as in the ABI encoding, and the length
+   of an array is not encoded.
+ - The direct arguments of ``abi.encodePacked`` are encoded without padding,
+   as long as they are not arrays (or ``string`` or ``bytes``).
+ - The encoding of an array is the concatenation of the
+   encoding of its elements **with** padding.
+ - Dynamically-sized types like ``string``, ``bytes`` or ``uint[]`` are encoded
+   without their length field.
+ - The encoding of ``string`` or ``bytes`` does not apply padding at the end
+   unless it is part of an array or struct (then it is padded to a multiple of
+   32 bytes).
 
 In general, the encoding is ambiguous as soon as there are two dynamically-sized elements,
 because of the missing length field.
@@ -636,3 +655,39 @@ If padding is needed, explicit type conversions can be used: ``abi.encodePacked(
 
 Since packed encoding is not used when calling functions, there is no special support
 for prepending a function selector. Since the encoding is ambiguous, there is no decoding function.
+
+.. warning::
+
+  If you use ``keccak256(abi.encodePacked(a, b))`` and both ``a`` and ``b`` are dynamic types,
+  it is easy to craft collisions in the hash value by moving parts of ``a`` into ``b`` and
+  vice-versa. More specifically, ``abi.encodePacked("a", "bc") == abi.encodePacked("ab", "c")``.
+  If you use ``abi.encodePacked`` for signatures, authentication or data integrity, make
+  sure to always use the same types and check that at most one of them is dynamic.
+  Unless there is a compelling reason, ``abi.encode`` should be preferred.
+
+
+.. _indexed_event_encoding:
+
+Encoding of Indexed Event Parameters
+====================================
+
+Indexed event parameters that are not value types, i.e. arrays and structs are not
+stored directly but instead a keccak256-hash of an encoding is stored. This encoding
+is defined as follows:
+
+ - the encoding of a ``bytes`` and ``string`` value is just the string contents
+   without any padding or length prefix.
+ - the encoding of a struct is the concatenation of the encoding of its members,
+   always padded to a multiple of 32 bytes (even ``bytes`` and ``string``).
+ - the encoding of an array (both dynamically- and statically-sized) is
+   the concatenation of the encoding of its elements, always padded to a multiple
+   of 32 bytes (even ``bytes`` and ``string``) and without any length prefix
+
+In the above, as usual, a negative number is padded by sign extension and not zero padded.
+``bytesNN`` types are padded on the right while ``uintNN`` / ``intNN`` are padded on the left.
+
+.. warning::
+
+    The encoding of a struct is ambiguous if it contains more than one dynamically-sized
+    array. Because of that, always re-check the event data and do not rely on the search result
+    based on the indexed parameters alone.

docs/bugs_by_version.json

@@ -624,5 +624,9 @@
     "0.5.3": {
         "bugs": [],
         "released": "2019-01-22"
+    },
+    "0.5.4": {
+        "bugs": [],
+        "released": "2019-02-12"
     }
 }