diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..47b6be3 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +3.7.2 \ No newline at end of file diff --git a/.github/workflows/envlogger-windows-build.yml b/.github/workflows/envlogger-windows-build.yml new file mode 100644 index 0000000..beeb8b0 --- /dev/null +++ b/.github/workflows/envlogger-windows-build.yml @@ -0,0 +1,69 @@ +name: envlogger-windows-build +on: + workflow_dispatch: + inputs: + python-x: + required: true + default: 3 + python-y: + required: true + default: 10 + build: + default: true + type: boolean + tag: + required: true + default: 1.0.8 +jobs: + build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + with: + repository: deepmind/envlogger + - uses: actions/checkout@v3 + with: + path: build + - uses: msys2/setup-msys2@v2 + with: + install: >- + gmp-devel + mingw-w64-x86_64-gcc + release: false + - run: | + echo "C:\msys64\usr\bin" >> $GITHUB_PATH + echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH + shell: bash + - uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-x }}.${{ inputs.python-y }} + - run: | + xcopy /Y build + xcopy build\patches patches + bash -c "git apply -v *.diff" + sed -i "s/cp310/cp${{ inputs.python-x }}${{ inputs.python-y }}/" envlogger\BUILD.bazel + sed -i "s/1\.0\.8/${{ inputs.tag }}/" envlogger\BUILD.bazel + curl -O https://storage.googleapis.com/google-code-archive-source/v2/code.google.com/mman-win32/source-archive.zip + C:\Windows\System32\tar.exe -xf source-archive.zip mman-win32/trunk + cd mman-win32\trunk + bash ./configure --libdir=/mingw64/lib --incdir=/mingw64/include/sys + make + make install + shell: cmd + - run: | + bazel test --compiler=mingw-gcc --define cpu=haswell --test_output=errors --verbose_failures envlogger/... + if: ${{ inputs.build == false }} + env: + BAZEL_SH: C:\msys64\usr\bin\bash.exe + shell: cmd + - run: | + bazel build --compiler=mingw-gcc --define cpu=haswell --test_output=errors --verbose_failures envlogger:package + if: ${{ inputs.build == true }} + env: + BAZEL_SH: C:\msys64\usr\bin\bash.exe + shell: cmd + - run: | + gh release upload ${{ inputs.tag }} bazel-bin/envlogger/*.whl --clobber -R ${{ github.repository }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash \ No newline at end of file diff --git a/BUILD.diff b/BUILD.diff new file mode 100644 index 0000000..e682e03 --- /dev/null +++ b/BUILD.diff @@ -0,0 +1,56 @@ +diff --git a/envlogger/BUILD.bazel b/envlogger/BUILD.bazel +index 6881b77..4727880 100644 +--- a/envlogger/BUILD.bazel ++++ b/envlogger/BUILD.bazel +@@ -15,9 +15,40 @@ + # Build targets for environment logger. + load("@py_deps//:requirements.bzl", "requirement") + load("@rules_python//python:defs.bzl", "py_library") ++load("@rules_python//python:packaging.bzl", "py_package", "py_wheel") + + package(default_visibility = ["//visibility:public"]) + ++py_package( ++ name = "pkg", ++ packages = ["envlogger"], ++ deps = [ ++ ":envlogger", ++ "//envlogger/backends:rlds_utils", ++ "//envlogger/backends:tfds_backend_writer", ++ "//envlogger/testing:catch_env", ++ ], ++) ++ ++py_wheel( ++ name = "package", ++ abi = "cp310", ++ distribution = "envlogger", ++ platform = "win_amd64", ++ python_tag = "cp310", ++ requires = [ ++ "absl-py", ++ "dm_env", ++ "numpy", ++ "protobuf>=3.14, <4.0", ++ "setuptools!=50.0.0", ++ ], ++ version = "1.0.8", ++ deps = [ ++ ":pkg", ++ ], ++) ++ + py_library( + name = "envlogger", + srcs = ["__init__.py"], +@@ -80,8 +111,8 @@ py_library( + name = "reader", + srcs = ["reader.py"], + data = [ +- "//envlogger/backends/python:episode_info.so", +- "//envlogger/backends/python:riegeli_dataset_reader.so", ++ "//envlogger/backends/python:episode_info.pyd", ++ "//envlogger/backends/python:riegeli_dataset_reader.pyd", + ], + deps = [ + ":step_data", diff --git a/BUILD2.diff b/BUILD2.diff new file mode 100644 index 0000000..d952c82 --- /dev/null +++ b/BUILD2.diff @@ -0,0 +1,42 @@ +diff --git a/envlogger/backends/BUILD.bazel b/envlogger/backends/BUILD.bazel +index 740d48e..46c190f 100644 +--- a/envlogger/backends/BUILD.bazel ++++ b/envlogger/backends/BUILD.bazel +@@ -47,7 +47,7 @@ py_test( + py_library( + name = "backend_reader", + srcs = ["backend_reader.py"], +- data = ["//envlogger/backends/python:episode_info.so"], ++ data = ["//envlogger/backends/python:episode_info.pyd"], + deps = [ + "//envlogger:step_data", + "//envlogger/converters:codec", +@@ -66,7 +66,7 @@ py_library( + py_library( + name = "in_memory_backend", + srcs = ["in_memory_backend.py"], +- data = ["//envlogger/backends/python:episode_info.so"], ++ data = ["//envlogger/backends/python:episode_info.pyd"], + deps = [ + ":backend_reader", + ":backend_writer", +@@ -92,8 +92,8 @@ py_library( + name = "riegeli_backend_reader", + srcs = ["riegeli_backend_reader.py"], + data = [ +- "//envlogger/backends/python:episode_info.so", +- "//envlogger/backends/python:riegeli_dataset_reader.so", ++ "//envlogger/backends/python:episode_info.pyd", ++ "//envlogger/backends/python:riegeli_dataset_reader.pyd", + ], + deps = [ + ":backend_reader", +@@ -109,7 +109,7 @@ py_library( + name = "riegeli_backend_writer", + srcs = ["riegeli_backend_writer.py"], + data = [ +- "//envlogger/backends/python:riegeli_dataset_writer.so", ++ "//envlogger/backends/python:riegeli_dataset_writer.pyd", + ], + deps = [ + ":backend_writer", diff --git a/BUILD3.diff b/BUILD3.diff new file mode 100644 index 0000000..eb124ba --- /dev/null +++ b/BUILD3.diff @@ -0,0 +1,85 @@ +diff --git a/envlogger/backends/python/BUILD.bazel b/envlogger/backends/python/BUILD.bazel +index 2b29d51..291b146 100644 +--- a/envlogger/backends/python/BUILD.bazel ++++ b/envlogger/backends/python/BUILD.bazel +@@ -19,6 +19,14 @@ package(default_visibility = ["//envlogger:__subpackages__"]) + + pybind_extension( + name = "riegeli_dataset_reader", ++ linkopts = [ ++ "-static-libgcc", ++ "-static-libstdc++", ++ "-Wl,-Bstatic,--whole-archive", ++ "-lwinpthread", ++ "-Wl,--no-whole-archive", ++ "-Wl,--allow-multiple-definition" ++ ], + srcs = ["riegeli_dataset_reader.cc"], + deps = [ + "//envlogger/backends/cc:riegeli_dataset_reader", +@@ -29,8 +37,23 @@ pybind_extension( + ], + ) + ++genrule( ++ name = "riegeli_dataset_reader_pyd", ++ srcs = ["riegeli_dataset_reader.so"], ++ outs = ["riegeli_dataset_reader.pyd"], ++ cmd = "mv $(location riegeli_dataset_reader.so) $(location riegeli_dataset_reader.pyd)", ++) ++ + pybind_extension( + name = "riegeli_dataset_writer", ++ linkopts = [ ++ "-static-libgcc", ++ "-static-libstdc++", ++ "-Wl,-Bstatic,--whole-archive", ++ "-lwinpthread", ++ "-Wl,--no-whole-archive", ++ "-Wl,--allow-multiple-definition" ++ ], + srcs = ["riegeli_dataset_writer.cc"], + deps = [ + "//envlogger/backends/cc:riegeli_dataset_writer", +@@ -41,8 +64,22 @@ pybind_extension( + ], + ) + ++genrule( ++ name = "riegeli_dataset_writer_pyd", ++ srcs = ["riegeli_dataset_writer.so"], ++ outs = ["riegeli_dataset_writer.pyd"], ++ cmd = "mv $(location riegeli_dataset_writer.so) $(location riegeli_dataset_writer.pyd)", ++) ++ + pybind_extension( + name = "episode_info", ++ linkopts = [ ++ "-static-libgcc", ++ "-static-libstdc++", ++ "-Wl,-Bstatic,--whole-archive", ++ "-lwinpthread", ++ "-Wl,--no-whole-archive" ++ ], + srcs = ["episode_info.cc"], + deps = [ + "//envlogger/backends/cc:episode_info", +@@ -51,10 +88,17 @@ pybind_extension( + ], + ) + ++genrule( ++ name = "episode_info_pyd", ++ srcs = ["episode_info.so"], ++ outs = ["episode_info.pyd"], ++ cmd = "mv $(location episode_info.so) $(location episode_info.pyd)", ++) ++ + py_test( + name = "episode_info_test", + srcs = ["episode_info_test.py"], +- data = [":episode_info.so"], ++ data = [":episode_info.pyd"], + deps = [ + "//envlogger/proto:storage_py_pb2", + requirement("absl-py"), diff --git a/BUILD4.diff b/BUILD4.diff new file mode 100644 index 0000000..5feed75 --- /dev/null +++ b/BUILD4.diff @@ -0,0 +1,12 @@ +diff --git a/envlogger/converters/BUILD.bazel b/envlogger/converters/BUILD.bazel +index 6670874..857750b 100644 +--- a/envlogger/converters/BUILD.bazel ++++ b/envlogger/converters/BUILD.bazel +@@ -81,6 +81,7 @@ cc_test( + name = "xtensor_codec_test", + srcs = ["xtensor_codec_test.cc"], + deps = [ ++ "@dbghelp", + ":make_visitor", + ":xtensor_codec", + "//envlogger/platform:parse_text_proto", diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..852f718 --- /dev/null +++ b/LICENSE @@ -0,0 +1,205 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Vadim Kuznetsov + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +This product bundles pread and pwrite from glog 0.5.0, which is available under +a "3-clause BSD" license. For details, see patches/fd_reader.diff and +patches/fd_writer.diff. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..d1f2976 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +[deepmind/envlogger](https://github.com/deepmind/envlogger) Windows Build (Beta) + +### Installation +Open the latest release, download a wheel for your Python version and install it using pip. For example, for version 1.0.8 and Python 3.10 run: +``` +pip install envlogger-1.0.8-cp310-cp310-win_amd64.whl +``` \ No newline at end of file diff --git a/WORKSPACE.diff b/WORKSPACE.diff new file mode 100644 index 0000000..6187f6d --- /dev/null +++ b/WORKSPACE.diff @@ -0,0 +1,95 @@ +diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel +index b8bafe5..5573aca 100644 +--- a/WORKSPACE.bazel ++++ b/WORKSPACE.bazel +@@ -43,8 +43,11 @@ http_archive( + # Install the requirements. + http_archive( + name = "rules_python", +- sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f", +- url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz", ++ sha256 = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea", ++ urls = [ ++ "https://mirror.bazel.build/github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz", ++ "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz", ++ ], + ) + + load("@rules_python//python:pip.bzl", "pip_install") +@@ -75,6 +78,8 @@ rules_proto_toolchains() + # abseil-cpp. + http_archive( + name = "com_google_absl", ++ patch_args = ["-p2"], ++ patches = ["//patches:bcrypt.diff"], + sha256 = "35f22ef5cb286f09954b7cc4c85b5a3f6221c9d4df6b8c4a1e9d399555b366ee", # SHARED_ABSL_SHA + strip_prefix = "abseil-cpp-997aaf3a28308eba1b9156aa35ab7bca9688e9f6", + urls = [ +@@ -97,6 +102,16 @@ http_archive( + # Riegeli. + http_archive( + name = "com_google_riegeli", ++ patch_args = ["-p2"], ++ patches = [ ++ "//patches:dbghelp.diff", ++ "//patches:fd_reader.diff", ++ "//patches:fd_reader_mode.diff", ++ "//patches:fd_writer.diff", ++ "//patches:fd_writer_mode.diff", ++ "//patches:mman.diff", ++ "//patches:py_extension.diff", ++ ], + sha256 = "59f94348f425fe3d7edd8f98ee0277ee2f4adecf7cf86f858f75039ce4955244", + strip_prefix = "riegeli-e9ba0d79a76b9fa1f0de46c11c464141e345338f", + url = "https://github.com/google/riegeli/archive/e9ba0d79a76b9fa1f0de46c11c464141e345338f.tar.gz", # 2022-06-27 +@@ -174,7 +189,7 @@ http_archive( + new_local_repository( + name = "gmp", + build_file = "//patches:gmp.BUILD.bazel", +- path = "/usr/include/", ++ path = "C:/msys64/mingw64/include", + ) + + # Google Logging library. +@@ -187,6 +202,8 @@ http_archive( + + http_archive( + name = "com_github_google_glog", ++ patch_args = ["-p2"], ++ patches = ["//patches:signalhandler.diff"], + sha256 = "21bc744fb7f2fa701ee8db339ded7dce4f975d0d55837a97be7d46e8382dea5a", + strip_prefix = "glog-0.5.0", + urls = ["https://github.com/google/glog/archive/v0.5.0.zip"], +@@ -224,6 +241,32 @@ git_repository( + git_repository( + name = "com_google_benchmark", + commit = "e991355c02b93fe17713efe04cbc2e278e00fdbd", # v.1.5.5 ++ patch_args = ["-p2"], ++ patches = ["//patches:shlwapi.diff"], + remote = "https://github.com/google/benchmark.git", + shallow_since = "1623228732 +0300", + ) ++ ++new_local_repository( ++ name = "bcrypt", ++ build_file = "//patches:bcrypt.BUILD.bazel", ++ path = "C:/msys64/mingw64/include", ++) ++ ++new_local_repository( ++ name = "dbghelp", ++ build_file = "//patches:dbghelp.BUILD.bazel", ++ path = "C:/msys64/mingw64/include", ++) ++ ++new_local_repository( ++ name = "mman", ++ build_file = "//patches:mman.BUILD.bazel", ++ path = "C:/msys64/mingw64/include", ++) ++ ++new_local_repository( ++ name = "shlwapi", ++ build_file = "//patches:shlwapi.BUILD.bazel", ++ path = "C:/msys64/mingw64/include", ++) diff --git a/bazelrc.diff b/bazelrc.diff new file mode 100644 index 0000000..f5348c2 --- /dev/null +++ b/bazelrc.diff @@ -0,0 +1,10 @@ +diff --git a/.bazelrc b/.bazelrc +index 3402023..06f0091 100644 +--- a/.bazelrc ++++ b/.bazelrc +@@ -20,3 +20,5 @@ build -c opt + build --cxxopt="-mavx" + build --cxxopt="-std=c++17" + build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" ++build --copt="-Wno-error=vla-parameter ++build --copt="-DHAVE_PTHREAD" diff --git a/cc_reader.diff b/cc_reader.diff new file mode 100644 index 0000000..5a40a4b --- /dev/null +++ b/cc_reader.diff @@ -0,0 +1,31 @@ +diff --git a/envlogger/backends/cross_language_test/cc_reader.cc b/envlogger/backends/cross_language_test/cc_reader.cc +index a86e3bd..ad77415 100644 +--- a/envlogger/backends/cross_language_test/cc_reader.cc ++++ b/envlogger/backends/cross_language_test/cc_reader.cc +@@ -27,7 +27,7 @@ + #include "envlogger/proto/storage.pb.h" + #include "xtensor/xarray.hpp" + +-ABSL_FLAG(std::string, trajectories_dir, "", "Path to reader trajectory."); ++//ABSL_FLAG(std::string, trajectories_dir, "", "Path to reader trajectory."); + + using ::testing::DoubleEq; + using ::testing::Eq; +@@ -37,13 +37,14 @@ using ::testing::Not; + using ::testing::SizeIs; + + int main(int argc, char** argv) { +- absl::ParseCommandLine(argc, argv); ++ //absl::ParseCommandLine(argc, argv); + + VLOG(0) << "Starting C++ Reader..."; +- VLOG(0) << "--trajectories_dir: " << absl::GetFlag(FLAGS_trajectories_dir); ++ //VLOG(0) << "--trajectories_dir: " << absl::GetFlag(FLAGS_trajectories_dir); + envlogger::RiegeliDatasetReader reader; + const absl::Status init_status = +- reader.Init(absl::GetFlag(FLAGS_trajectories_dir)); ++ //reader.Init(absl::GetFlag(FLAGS_trajectories_dir)); ++ reader.Init(argv[1]); + VLOG(0) << "init_status: " << init_status; + + VLOG(0) << "reader.NumSteps(): " << reader.NumSteps(); diff --git a/codec_test.diff b/codec_test.diff new file mode 100644 index 0000000..332d69f --- /dev/null +++ b/codec_test.diff @@ -0,0 +1,20 @@ +diff --git a/envlogger/converters/codec_test.py b/envlogger/converters/codec_test.py +index ed870ad..cac805a 100644 +--- a/envlogger/converters/codec_test.py ++++ b/envlogger/converters/codec_test.py +@@ -21,6 +21,15 @@ from envlogger.converters import codec + from envlogger.proto import storage_pb2 + import numpy as np + ++# Patch ++array_old = np.array ++def array_new(object, **kwargs): ++ r = array_old(object, **kwargs) ++ if "dtype" not in kwargs and r.dtype == np.int32: ++ return array_old(object, dtype=np.int64, **kwargs) ++ return r ++np.array = array_new ++ + + class NumpyConvertersTest(parameterized.TestCase): + diff --git a/cross_language_test.diff b/cross_language_test.diff new file mode 100644 index 0000000..761671f --- /dev/null +++ b/cross_language_test.diff @@ -0,0 +1,36 @@ +diff --git a/envlogger/backends/cross_language_test/cross_language_test.py b/envlogger/backends/cross_language_test/cross_language_test.py +index 43dd638..9685a94 100644 +--- a/envlogger/backends/cross_language_test/cross_language_test.py ++++ b/envlogger/backends/cross_language_test/cross_language_test.py +@@ -28,9 +28,11 @@ from rules_python.python.runfiles import runfiles + + def _execute_binary(rel_path: str, args: Sequence[str]) -> bytes: + r = runfiles.Create() +- path = r.Rlocation(os.path.join('__main__', 'envlogger', rel_path)) ++ path = r.Rlocation(os.path.join('__main__', 'envlogger', rel_path).replace("\\", "/")) + cmd = [path] + args +- return subprocess.check_output(cmd, env=r.EnvVars()) ++ env_vars = r.EnvVars() ++ env_vars.update(os.environ) ++ return subprocess.check_output(cmd, env=env_vars) + + + class CrossLanguageTest(parameterized.TestCase): +@@ -43,14 +45,14 @@ class CrossLanguageTest(parameterized.TestCase): + + # Find Python writer and run it. + py_writer_output = _execute_binary( +- 'backends/cross_language_test/py_writer', ++ 'backends/cross_language_test/py_writer.exe', + args=[f'--trajectories_dir={trajectories_dir}']) + logging.info('py_writer_output: %r', py_writer_output) + + # Find C++ reader and run it. + cc_reader_output = _execute_binary( +- 'backends/cross_language_test/cc_reader', +- args=[f'--trajectories_dir={trajectories_dir}']) ++ 'backends/cross_language_test/cc_reader.exe', ++ args=[f'{trajectories_dir}']) + logging.info('cc_reader_output: %r', cc_reader_output) + + # If everything went well, there should be no diff --git a/filesystem.diff b/filesystem.diff new file mode 100644 index 0000000..64dc08b --- /dev/null +++ b/filesystem.diff @@ -0,0 +1,22 @@ +diff --git a/envlogger/platform/default/filesystem.cc b/envlogger/platform/default/filesystem.cc +index 143fce1..08a381b 100644 +--- a/envlogger/platform/default/filesystem.cc ++++ b/envlogger/platform/default/filesystem.cc +@@ -26,7 +26,7 @@ namespace envlogger { + namespace file { + + std::string JoinPath(absl::string_view dirname, absl::string_view basename) { +- return std::filesystem::path(dirname) / basename; ++ return (std::filesystem::path(dirname) / basename).string(); + } + + absl::Status CreateDir(absl::string_view path) { +@@ -46,7 +46,7 @@ absl::StatusOr> GetSubdirectories( + !std::filesystem::exists(entry.path() / sentinel)) { + continue; + } +- subdirs.push_back(entry.path()); ++ subdirs.push_back(entry.path().string()); + } + } + return subdirs; diff --git a/highwayhash.diff b/highwayhash.diff new file mode 100644 index 0000000..41a825a --- /dev/null +++ b/highwayhash.diff @@ -0,0 +1,13 @@ +diff --git a/patches/highwayhash.BUILD.bazel b/patches/highwayhash.BUILD.bazel +index 98d0b55..83f23c4 100644 +--- a/patches/highwayhash.BUILD.bazel ++++ b/patches/highwayhash.BUILD.bazel +@@ -18,7 +18,7 @@ licenses(["notice"]) + + config_setting( + name = "haswell", +- values = {"cpu": "haswell"}, ++ define_values = {"cpu": "haswell"}, + ) + + config_setting( diff --git a/patches/bcrypt.BUILD.bazel b/patches/bcrypt.BUILD.bazel new file mode 100644 index 0000000..5dc6341 --- /dev/null +++ b/patches/bcrypt.BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "bcrypt", + hdrs = ["bcrypt.h"], + includes = ["."], + linkopts = [ + "-lbcrypt", + ], + visibility = ["//visibility:public"], +) diff --git a/patches/bcrypt.diff b/patches/bcrypt.diff new file mode 100644 index 0000000..010e818 --- /dev/null +++ b/patches/bcrypt.diff @@ -0,0 +1,12 @@ +diff --git a/com_google_absl/absl/random/BUILD.bazel b/com_google_absl/absl/random/BUILD.bazel +index 66ffcbc..09f8755 100644 +--- a/com_google_absl/absl/random/BUILD.bazel ++++ b/com_google_absl/absl/random/BUILD.bazel +@@ -34,6 +34,7 @@ cc_library( + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ ++ "@bcrypt", + ":distributions", + ":seed_sequences", + "//absl/random/internal:nonsecure_base", diff --git a/patches/dbghelp.BUILD.bazel b/patches/dbghelp.BUILD.bazel new file mode 100644 index 0000000..3e90437 --- /dev/null +++ b/patches/dbghelp.BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "dbghelp", + hdrs = ["dbghelp.h"], + includes = ["."], + linkopts = [ + "-ldbghelp", + ], + visibility = ["//visibility:public"], +) diff --git a/patches/dbghelp.diff b/patches/dbghelp.diff new file mode 100644 index 0000000..a8989d5 --- /dev/null +++ b/patches/dbghelp.diff @@ -0,0 +1,20 @@ +diff --git a/com_google_riegeli/riegeli/records/BUILD b/com_google_riegeli/riegeli/records/BUILD +index 4c344f7..e208dc6 100644 +--- a/com_google_riegeli/riegeli/records/BUILD ++++ b/com_google_riegeli/riegeli/records/BUILD +@@ -12,6 +12,7 @@ cc_library( + srcs = ["record_reader.cc"], + hdrs = ["record_reader.h"], + deps = [ ++ "@dbghelp", + ":chunk_reader", + ":record_position", + ":records_metadata_cc_proto", +@@ -45,6 +46,7 @@ cc_library( + srcs = ["record_writer.cc"], + hdrs = ["record_writer.h"], + deps = [ ++ "@dbghelp", + ":chunk_writer", + ":record_position", + ":records_metadata_cc_proto", diff --git a/patches/fd_reader.diff b/patches/fd_reader.diff new file mode 100644 index 0000000..c66e680 --- /dev/null +++ b/patches/fd_reader.diff @@ -0,0 +1,52 @@ +diff --git a/com_google_riegeli/riegeli/bytes/fd_reader.cc b/com_google_riegeli/riegeli/bytes/fd_reader.cc +index bf0b6ce..28328e7 100644 +--- a/com_google_riegeli/riegeli/bytes/fd_reader.cc ++++ b/com_google_riegeli/riegeli/bytes/fd_reader.cc +@@ -1,3 +1,47 @@ ++// Copyright (c) 1999, Google Inc. ++// All rights reserved. ++// ++// Redistribution and use in source and binary forms, with or without ++// modification, are permitted provided that the following conditions are ++// met: ++// ++// * Redistributions of source code must retain the above copyright ++// notice, this list of conditions and the following disclaimer. ++// * Redistributions in binary form must reproduce the above ++// copyright notice, this list of conditions and the following disclaimer ++// in the documentation and/or other materials provided with the ++// distribution. ++// * Neither the name of Google Inc. nor the names of its ++// contributors may be used to endorse or promote products derived from ++// this software without specific prior written permission. ++// ++// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++#include ++ ++static ssize_t pread(int fd, void* buf, size_t count, off_t offset) { ++ off_t orig_offset = lseek(fd, 0, SEEK_CUR); ++ if (orig_offset == (off_t)-1) ++ return -1; ++ if (lseek(fd, offset, SEEK_CUR) == (off_t)-1) ++ return -1; ++ ssize_t len = read(fd, buf, count); ++ if (len < 0) ++ return len; ++ if (lseek(fd, orig_offset, SEEK_SET) == (off_t)-1) ++ return -1; ++ return len; ++} + // Copyright 2017 Google LLC + // + // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/patches/fd_reader_mode.diff b/patches/fd_reader_mode.diff new file mode 100644 index 0000000..b308b94 --- /dev/null +++ b/patches/fd_reader_mode.diff @@ -0,0 +1,13 @@ +diff --git a/com_google_riegeli/riegeli/bytes/fd_reader.h b/com_google_riegeli/riegeli/bytes/fd_reader.h +index 70fd051..721f9a3 100644 +--- a/com_google_riegeli/riegeli/bytes/fd_reader.h ++++ b/com_google_riegeli/riegeli/bytes/fd_reader.h +@@ -154,7 +154,7 @@ class FdReaderBase : public BufferedReader { + + private: + absl::optional assumed_filename_; +- int mode_ = O_RDONLY; ++ int mode_ = O_RDONLY | O_BINARY; + absl::optional assumed_pos_; + absl::optional independent_pos_; + bool growing_source_ = false; diff --git a/patches/fd_writer.diff b/patches/fd_writer.diff new file mode 100644 index 0000000..07e62a8 --- /dev/null +++ b/patches/fd_writer.diff @@ -0,0 +1,73 @@ +diff --git a/com_google_riegeli/riegeli/bytes/fd_writer.cc b/com_google_riegeli/riegeli/bytes/fd_writer.cc +index 2e2c751..1dbdd9a 100644 +--- a/com_google_riegeli/riegeli/bytes/fd_writer.cc ++++ b/com_google_riegeli/riegeli/bytes/fd_writer.cc +@@ -1,3 +1,47 @@ ++// Copyright (c) 1999, Google Inc. ++// All rights reserved. ++// ++// Redistribution and use in source and binary forms, with or without ++// modification, are permitted provided that the following conditions are ++// met: ++// ++// * Redistributions of source code must retain the above copyright ++// notice, this list of conditions and the following disclaimer. ++// * Redistributions in binary form must reproduce the above ++// copyright notice, this list of conditions and the following disclaimer ++// in the documentation and/or other materials provided with the ++// distribution. ++// * Neither the name of Google Inc. nor the names of its ++// contributors may be used to endorse or promote products derived from ++// this software without specific prior written permission. ++// ++// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++#include ++ ++static ssize_t pwrite(int fd, const void* buf, size_t count, off_t offset) { ++ off_t orig_offset = lseek(fd, 0, SEEK_CUR); ++ if (orig_offset == (off_t)-1) ++ return -1; ++ if (lseek(fd, offset, SEEK_CUR) == (off_t)-1) ++ return -1; ++ ssize_t len = write(fd, buf, count); ++ if (len < 0) ++ return len; ++ if (lseek(fd, orig_offset, SEEK_SET) == (off_t)-1) ++ return -1; ++ return len; ++} + // Copyright 2017 Google LLC + // + // Licensed under the Apache License, Version 2.0 (the "License"); +@@ -12,6 +56,10 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + ++#define WIN32_LEAN_AND_MEAN ++#include ++#define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) ++ + // Make `pwrite()` and `ftruncate()` available. + #if !defined(_XOPEN_SOURCE) || _XOPEN_SOURCE < 500 + #undef _XOPEN_SOURCE +@@ -88,7 +136,8 @@ inline void FdWriterBase::InitializePos( + if (assumed_pos == absl::nullopt) { + // Flags are needed only if `assumed_pos == absl::nullopt`. Avoid `fcntl()` + // otherwise. +- flags = fcntl(dest, F_GETFL); ++ //flags = fcntl(dest, F_GETFL); ++ flags = O_RDWR; + if (ABSL_PREDICT_FALSE(flags < 0)) { + FailOperation("fcntl()"); + return; diff --git a/patches/fd_writer_mode.diff b/patches/fd_writer_mode.diff new file mode 100644 index 0000000..ec53ecf --- /dev/null +++ b/patches/fd_writer_mode.diff @@ -0,0 +1,13 @@ +diff --git a/com_google_riegeli/riegeli/bytes/fd_writer.h b/com_google_riegeli/riegeli/bytes/fd_writer.h +index ada2bc1..533ad99 100644 +--- a/com_google_riegeli/riegeli/bytes/fd_writer.h ++++ b/com_google_riegeli/riegeli/bytes/fd_writer.h +@@ -158,7 +158,7 @@ class FdWriterBase : public BufferedWriter { + + private: + absl::optional assumed_filename_; +- int mode_ = O_WRONLY | O_CREAT | O_TRUNC; ++ int mode_ = O_WRONLY | O_CREAT | O_TRUNC | O_BINARY; + mode_t permissions_ = 0666; + absl::optional assumed_pos_; + absl::optional independent_pos_; diff --git a/patches/mman.BUILD.bazel b/patches/mman.BUILD.bazel new file mode 100644 index 0000000..a30177a --- /dev/null +++ b/patches/mman.BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "mman", + hdrs = ["sys/mman.h"], + includes = ["."], + linkopts = [ + "-lmman", + ], + visibility = ["//visibility:public"], +) diff --git a/patches/mman.diff b/patches/mman.diff new file mode 100644 index 0000000..2085c50 --- /dev/null +++ b/patches/mman.diff @@ -0,0 +1,12 @@ +diff --git a/com_google_riegeli/riegeli/bytes/BUILD b/com_google_riegeli/riegeli/bytes/BUILD +index c625b35..a290898 100644 +--- a/com_google_riegeli/riegeli/bytes/BUILD ++++ b/com_google_riegeli/riegeli/bytes/BUILD +@@ -622,6 +622,7 @@ cc_library( + # files provide. + features = ["-use_header_modules"], + deps = [ ++ "@mman", + ":buffer_options", + ":buffered_reader", + ":chain_reader", diff --git a/patches/py_extension.diff b/patches/py_extension.diff new file mode 100644 index 0000000..6ea762d --- /dev/null +++ b/patches/py_extension.diff @@ -0,0 +1,35 @@ +diff --git a/com_google_riegeli/python/riegeli/py_extension.bzl b/com_google_riegeli/python/riegeli/py_extension.bzl +index fdefce3..31ba938 100644 +--- a/com_google_riegeli/python/riegeli/py_extension.bzl ++++ b/com_google_riegeli/python/riegeli/py_extension.bzl +@@ -39,14 +39,29 @@ def py_extension( + + native.cc_binary( + name = cc_binary_name, ++ linkopts = [ ++ "-static-libgcc", ++ "-static-libstdc++", ++ "-Wl,-Bstatic,--whole-archive", ++ "-lwinpthread", ++ "-Wl,--no-whole-archive", ++ "-Wl,--allow-multiple-definition" ++ ], + linkshared = True, + linkstatic = True, + visibility = ["//visibility:private"], + deps = [cc_library_name], + ) + ++ native.genrule( ++ name = name + "_pyd", ++ srcs = [cc_binary_name], ++ outs = [name + ".pyd"], ++ cmd = "mv $(location " + cc_binary_name + ") $(location " + name + ".pyd)", ++ ) ++ + native.py_library( + name = name, +- data = [cc_binary_name], ++ data = [name + ".pyd"], + visibility = visibility, + ) diff --git a/patches/shlwapi.BUILD.bazel b/patches/shlwapi.BUILD.bazel new file mode 100644 index 0000000..f4c9fa8 --- /dev/null +++ b/patches/shlwapi.BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "shlwapi", + hdrs = ["shlwapi.h"], + includes = ["."], + linkopts = [ + "-lshlwapi", + ], + visibility = ["//visibility:public"], +) diff --git a/patches/shlwapi.diff b/patches/shlwapi.diff new file mode 100644 index 0000000..481e044 --- /dev/null +++ b/patches/shlwapi.diff @@ -0,0 +1,12 @@ +diff --git a/com_google_benchmark/BUILD.bazel b/com_google_benchmark/BUILD.bazel +index eb35b62..fc96204 100644 +--- a/com_google_benchmark/BUILD.bazel ++++ b/com_google_benchmark/BUILD.bazel +@@ -12,6 +12,7 @@ config_setting( + + cc_library( + name = "benchmark", ++ deps = ["@shlwapi"], + srcs = glob( + [ + "src/*.cc", diff --git a/patches/signalhandler.diff b/patches/signalhandler.diff new file mode 100644 index 0000000..9c9c7c4 --- /dev/null +++ b/patches/signalhandler.diff @@ -0,0 +1,13 @@ +diff --git a/com_github_google_glog/src/signalhandler.cc b/com_github_google_glog/src/signalhandler.cc +index b6d6e25..dc2b2f4 100644 +--- a/com_github_google_glog/src/signalhandler.cc ++++ b/com_github_google_glog/src/signalhandler.cc +@@ -303,7 +303,7 @@ void FailureSignalHandler(int signal_number, + // Another thread is dumping stuff. Let's wait until that thread + // finishes the job and kills the process. + while (true) { +- sleep(1); ++ Sleep(1000); + } + } + // This is the first time we enter the signal handler. We are going to diff --git a/riegeli_dataset_io_test.diff b/riegeli_dataset_io_test.diff new file mode 100644 index 0000000..12fdf1b --- /dev/null +++ b/riegeli_dataset_io_test.diff @@ -0,0 +1,12 @@ +diff --git a/envlogger/backends/cc/riegeli_dataset_io_test.cc b/envlogger/backends/cc/riegeli_dataset_io_test.cc +index 28a0af5..62aa282 100644 +--- a/envlogger/backends/cc/riegeli_dataset_io_test.cc ++++ b/envlogger/backends/cc/riegeli_dataset_io_test.cc +@@ -75,6 +75,7 @@ TEST(RiegeliDatasetTest, MetadataTest) { + const auto actual_metadata = reader.Metadata(); + EXPECT_THAT(actual_metadata, Not(Eq(std::nullopt))); + EXPECT_THAT(*actual_metadata, EqualsProto(metadata)); ++ reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } diff --git a/riegeli_dataset_reader_test.diff b/riegeli_dataset_reader_test.diff new file mode 100644 index 0000000..fad2db6 --- /dev/null +++ b/riegeli_dataset_reader_test.diff @@ -0,0 +1,55 @@ +diff --git a/envlogger/backends/cc/riegeli_dataset_reader_test.cc b/envlogger/backends/cc/riegeli_dataset_reader_test.cc +index 9add0fb..d27adb3 100644 +--- a/envlogger/backends/cc/riegeli_dataset_reader_test.cc ++++ b/envlogger/backends/cc/riegeli_dataset_reader_test.cc +@@ -160,6 +160,7 @@ TEST(DataDirectoryIndex, SimpleMetadataAndSpec) { + const auto actual_metadata = reader.Metadata(); + EXPECT_THAT(actual_metadata, Not(Eq(std::nullopt))); + EXPECT_THAT(*actual_metadata, EqualsProto(metadata)); ++ reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } +@@ -223,6 +224,7 @@ TEST(DataDirectoryIndex, OneShard) { + EXPECT_THAT(episodes, ElementsAre(EqualsEpisode(0, 3), EqualsEpisode(3, 2))); + EXPECT_THAT(episodes[0].metadata, Not(Eq(std::nullopt))); + EXPECT_THAT(*episodes[0].metadata, EqualsProto(episode0_metadata)); ++ reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } +@@ -286,6 +288,7 @@ TEST(DataDirectoryIndex, OneShardNonDmDataPayload) { + EXPECT_THAT(episodes, ElementsAre(EqualsEpisode(0, 3), EqualsEpisode(3, 2))); + EXPECT_THAT(episodes[0].metadata, Not(Eq(std::nullopt))); + EXPECT_THAT(*episodes[0].metadata, EqualsProto(episode0_metadata)); ++ reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } +@@ -348,6 +351,7 @@ TEST(DataDirectoryIndex, TwoShards) { + } + EXPECT_THAT(episodes, ElementsAre(EqualsEpisode(0, 3), EqualsEpisode(3, 2), + EqualsEpisode(5, 1), EqualsEpisode(6, 3))); ++ reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } +@@ -418,6 +422,9 @@ TEST(DataDirectoryIndex, GetShardTest) { + episode1_steps, + ElementsAre(EqualsProto("datum: { values: { float_values: 4.0}}"), + EqualsProto("datum: { values: { float_values: 5.0}}"))); ++ reader.Close(); ++ episode0->Close(); ++ episode1->Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } +@@ -483,6 +490,8 @@ TEST(DataDirectoryIndex, CloneTest) { + EqualsProto("datum: { values: { float_values: 3.0}}"), + EqualsProto("datum: { values: { float_values: 4.0}}"), + EqualsProto("datum: { values: { float_values: 5.0}}"))); ++ reader.Close(); ++ clone->Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } diff --git a/riegeli_dataset_writer_test.diff b/riegeli_dataset_writer_test.diff new file mode 100644 index 0000000..7a78b6a --- /dev/null +++ b/riegeli_dataset_writer_test.diff @@ -0,0 +1,12 @@ +diff --git a/envlogger/backends/cc/riegeli_dataset_writer_test.cc b/envlogger/backends/cc/riegeli_dataset_writer_test.cc +index 1088cb7..f7c6404 100644 +--- a/envlogger/backends/cc/riegeli_dataset_writer_test.cc ++++ b/envlogger/backends/cc/riegeli_dataset_writer_test.cc +@@ -200,6 +200,7 @@ TEST(RiegeliDatasetWriterTest, EpisodeMetadata) { + ElementsAre(EqualsProto(episode0_metadata), + // Second episode1 should have no metadata here. + EqualsProto(episode2_metadata))); ++ episode_metadata_reader.Close(); + + ENVLOGGER_EXPECT_OK(file::RecursivelyDelete(data_dir)); + } diff --git a/riegeli_shard_io_test.diff b/riegeli_shard_io_test.diff new file mode 100644 index 0000000..b50ac91 --- /dev/null +++ b/riegeli_shard_io_test.diff @@ -0,0 +1,27 @@ +diff --git a/envlogger/backends/cc/riegeli_shard_io_test.cc b/envlogger/backends/cc/riegeli_shard_io_test.cc +index 1050cec..78f8686 100644 +--- a/envlogger/backends/cc/riegeli_shard_io_test.cc ++++ b/envlogger/backends/cc/riegeli_shard_io_test.cc +@@ -33,7 +33,7 @@ + #include "envlogger/platform/test_macros.h" + #include "envlogger/proto/storage.pb.h" + +-ABSL_FLAG(bool, run_benchmarks, false, "Whether to run benchmarks."); ++//ABSL_FLAG(bool, run_benchmarks, false, "Whether to run benchmarks."); + + namespace envlogger { + namespace { +@@ -244,10 +244,11 @@ BENCHMARK(BM_WriteRead) + } // namespace envlogger + + int main(int argc, char **argv) { +- absl::ParseCommandLine(argc, argv); ++ //absl::ParseCommandLine(argc, argv); + ::testing::InitGoogleTest(&argc, argv); ++ /* + if (absl::GetFlag(FLAGS_run_benchmarks)) { + ::benchmark::RunSpecifiedBenchmarks(); +- } ++ }*/ + return RUN_ALL_TESTS(); + } diff --git a/tfds_backend_testlib.diff b/tfds_backend_testlib.diff new file mode 100644 index 0000000..f88eae3 --- /dev/null +++ b/tfds_backend_testlib.diff @@ -0,0 +1,13 @@ +diff --git a/envlogger/backends/tfds_backend_testlib.py b/envlogger/backends/tfds_backend_testlib.py +index c3b4df1..18f99d6 100644 +--- a/envlogger/backends/tfds_backend_testlib.py ++++ b/envlogger/backends/tfds_backend_testlib.py +@@ -76,7 +76,7 @@ def catch_env_tfds_config( + observation_info=tfds.features.Tensor( + shape=(10, 5), dtype=tf.float32, + encoding=tfds.features.Encoding.ZLIB), +- action_info=tf.int64, ++ action_info=tf.int32, + reward_info=tf.float64, + discount_info=tf.float64, + step_metadata_info={'timestamp': tf.int64},