For each line integration

CMakeLists.txt

@@ -26,9 +26,7 @@ find_package(Boost REQUIRED COMPONENTS
     unit_test_framework
     regex
     date_time
-    python312
     system
-    iostreams
 )
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})

arch/ansi.cc

@@ -0,0 +1,39 @@
+/* ansi.cc                                                          -*- C++ -*-
+   Jeremy Barnes, 21 February 2007
+   Copyright (c) 2007 Jeremy Barnes.  All rights reserved.
+   This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.
+
+   ANSI escape codes for pretty-printing on terminals.
+*/
+
+#include "ansi.h"
+#include <cstdlib>
+
+// From c standard library
+extern "C" {
+int isatty(int);
+}
+
+
+namespace MLDB {
+namespace ansi {
+
+static bool get_ansi_colors_enabled()
+{
+    std::string ansi_colors_env = ::getenv("NO_COLOR") ? ::getenv("NO_COLOR") : "AUTO";
+
+    if (ansi_colors_env == "NEVER") {
+        return true;
+    }
+    else if (ansi_colors_env == "AUTO") {
+        return ::isatty(0);
+    }
+    else {
+        return false;
+    }
+}
+
+const bool enable_ansi = get_ansi_colors_enabled();
+
+} // namespace ansi
+} // namespace MLDB

arch/ansi.h

@@ -0,0 +1,45 @@
+/* ansi.h                                                          -*- C++ -*-
+   Jeremy Barnes, 21 February 2007
+   Copyright (c) 2007 Jeremy Barnes.  All rights reserved.
+   This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.
+
+   ANSI escape codes for pretty-printing on terminals.
+*/
+
+
+#pragma once
+
+#include <iostream>
+
+namespace MLDB {
+namespace ansi {
+
+extern const bool enable_ansi;
+
+#define ANSI_CODE(name, sequence) \
+static constexpr const char ansi_color_##name [] = "\u001b" sequence "m\0"; \
+inline std::ostream & name(std::ostream & stream) { if (enable_ansi) stream << ansi_color_##name;  return stream; } \
+inline const char * ansi_str_##name() { return enable_ansi ? ansi_color_##name : ""; }
+
+ANSI_CODE(reset, "[0");
+ANSI_CODE(bold, "[1");
+ANSI_CODE(underline, "[4");
+ANSI_CODE(reversed, "[7");
+
+#define ANSI_COLOR(name, sequence) \
+ANSI_CODE(name, "[3" sequence) \
+ANSI_CODE(bright_##name, "[3" sequence ";1") \
+ANSI_CODE(bg_##name, "[4" sequence) \
+ANSI_CODE(bg_bright_##name, "[4" sequence ";1")
+
+ANSI_COLOR(black,   "0");
+ANSI_COLOR(red,     "1");
+ANSI_COLOR(green,   "2");
+ANSI_COLOR(yellow,  "3");
+ANSI_COLOR(blue,    "4");
+ANSI_COLOR(magenta, "5");
+ANSI_COLOR(cyan,    "6");
+ANSI_COLOR(white,   "7");
+
+} // namespace ansi
+} // namespace MLDB

arch/arch.mk

@@ -24,8 +24,9 @@ LIBARCH_SOURCES := \
         file_functions.cc \
 		wait_on_address.cc \
 		wakeup_fd.cc \
-		thread_specific.cc
-
+		thread_specific.cc \
+	ansi.cc \
+	missing_typeids.cc \
 
 ifeq ($(ARCH),x86_64)
 LIBARCH_SOURCES += simd_vector_avx.cc

arch/atomic_min_max.h

@@ -0,0 +1,33 @@
+// public domain
+
+#pragma once
+
+#include <atomic>
+
+namespace MLDB {
+
+template<typename T, typename T2>
+void atomic_min(std::atomic<T> & val, T2 other)
+{
+    T current = val.load(std::memory_order_relaxed);
+    for (;;) {
+        if (other >= current)
+            return;
+        if (val.compare_exchange_weak(current, other, std::memory_order_relaxed))
+            return;
+    }
+}
+
+template<typename T, typename T2>
+void atomic_max(std::atomic<T> & val, T2 other)
+{
+    T current = val.load(std::memory_order_relaxed);
+    for (;;) {
+        if (other <= current)
+            return;
+        if (val.compare_exchange_weak(current, other, std::memory_order_relaxed))
+            return;
+    }
+}
+
+} // namespace MLDB

arch/bit_range_ops.h

@@ -17,6 +17,7 @@
 #include <stdint.h>
 #include <algorithm>
 #include <atomic>
+#include <bit>
 
 
 namespace MLDB {
@@ -355,6 +356,34 @@ MLDB_ALWAYS_INLINE signed long long fixup_extract(signed long long e, shift_t bi
     return sign_extend(e, bits);
 }
 
+// Index of the nth set bit
+// https://stackoverflow.com/questions/7669057/find-nth-set-bit-in-an-int
+//__attribute__((target("default")))
+inline uint64_t nthSetBit(uint64_t v, unsigned n)
+{
+    if (n > (unsigned)__builtin_popcountll(v)/* std::popcount(v) */)
+        return 0;
+
+    for (unsigned i=0; n > 0 && i<n; i++) {
+        v &= v-1; // remove the least significant bit
+    }
+    return v & ~(v-1); // extract the least significant bit
+}
+
+#if 0
+__attribute__((target("bmi2")))
+inline uint64_t nthSetBit(uint64_t x, unsigned n)
+{
+    return _pdep_u64(1ULL << n, x);
+}
+#endif
+
+inline int nthSetBitIndex(uint64_t x, unsigned n)
+{
+    uint64_t bit = nthSetBit(x, n);
+    return bit == 0 ? -1 : std::countr_zero(bit);
+}
+
 /*****************************************************************************/
 /* MEM_BUFFER                                                                */
 /*****************************************************************************/

arch/missing_typeids.cc

@@ -0,0 +1,14 @@
+#include <typeinfo>
+#include <iostream>
+
+#ifdef __clang__
+#ifdef __APPLE__
+// Sometimes the typeinfo node for _Float16 is missing (llvm on OSX, at least)
+// Create it here if that's the case
+struct HalfTypeInfo: public std::type_info {
+    HalfTypeInfo(): std::type_info("Dh") {};
+};
+
+HalfTypeInfo _ZTIDF16_ __attribute__((__weak__));
+#endif /* __APPLE__ */
+#endif /* __clang__ */

arch/rtti_utils.cc

@@ -80,7 +80,8 @@ const void * is_convertible(const std::type_info & from_type,
             //cerr << "could upcast" << endl;
             return ur.dst_ptr;
         }
-
+        return nullptr;
+
         //cerr << "couldn't upcast" << endl;
 
         //const char * adj_ptr = (const char *)obj;

arch/testing/bit_range_ops_test.cc

@@ -453,3 +453,31 @@ BOOST_AUTO_TEST_CASE(testExtraBits)
         BOOST_CHECK_EQUAL(data[1], MINUS1);
     }        
 }
+
+BOOST_AUTO_TEST_CASE(nth_bit_index)
+{
+    BOOST_CHECK_EQUAL(nthSetBit(1, 0), 1);
+    BOOST_CHECK_EQUAL(nthSetBit(2, 0), 2);
+    BOOST_CHECK_EQUAL(nthSetBit(4, 0), 4);
+
+    BOOST_CHECK_EQUAL(nthSetBit(1, 1), 0);
+    BOOST_CHECK_EQUAL(nthSetBit(2, 1), 0);
+    BOOST_CHECK_EQUAL(nthSetBit(4, 1), 0);
+
+    BOOST_CHECK_EQUAL(nthSetBit(7, 0), 1);
+    BOOST_CHECK_EQUAL(nthSetBit(7, 1), 2);
+    BOOST_CHECK_EQUAL(nthSetBit(7, 2), 4);
+    BOOST_CHECK_EQUAL(nthSetBit(7, 3), 0);
+
+    BOOST_CHECK_EQUAL(nthSetBit(0, 64), 0);
+    BOOST_CHECK_EQUAL(nthSetBit(1, 64), 0);
+    BOOST_CHECK_EQUAL(nthSetBit(std::numeric_limits<uint64_t>::max(), 64), 0);
+
+    for (int i = 0;  i < 63;  ++i) {
+        BOOST_CHECK_EQUAL(nthSetBitIndex(1ULL<<i, 0), i);
+        BOOST_CHECK_EQUAL(nthSetBitIndex(1ULL<<i, 1), -1);
+        BOOST_CHECK_EQUAL(nthSetBitIndex(std::numeric_limits<uint64_t>::max(), i), i);
+    }
+    BOOST_CHECK_EQUAL(nthSetBitIndex(2, 0), 1);
+    BOOST_CHECK_EQUAL(nthSetBitIndex(3, 1), 1);
+}

arch/testing/vm_test.cc

@@ -15,6 +15,7 @@
 #include "mldb/utils/vector_utils.h"
 #include "mldb/arch/exception_handler.h"
 #include "mldb/base/scope.h"
+#include "mldb/arch/info.h"
 
 #include <boost/test/unit_test.hpp>
 #include <iostream>
@@ -30,15 +31,6 @@ using namespace std;
 
 using boost::unit_test::test_suite;
 
-// Copied from utils/info.cc due to not being able to include utils lib
-size_t num_open_files()
-{
-    size_t result = 0;
-    for (auto entry: std::filesystem::directory_iterator("/proc/self/fd"))
-        ++result;
-    return result;
-}
-
 void test_function()
 {
     cerr << "hello" << endl;

arch/vm.h

@@ -33,6 +33,18 @@ constexpr int page_offset_mask = page_size - 1;
 
 static const size_t page_num_mask = ~(size_t)page_offset_mask;
 
+// Takes the given number of bytes and rounds up to the multiple of page size
+// that fits it all in.
+inline size_t roundUpToPageSize(size_t mem)
+{
+    return (mem + page_size - 1) / page_size * page_size;
+}
+
+inline size_t roundDownToPageSize(size_t mem)
+{
+    return ssize_t(mem - page_size + 1) / page_size * page_size;
+}
+
 struct Pagemap_Entry {
 
     // Note that this just looks at the pfn.  The other flags might change

base/base.mk

@@ -4,10 +4,12 @@
 LIBBASE_SOURCES := \
     parse_context.cc \
 	thread_pool.cc \
+	compute_context.cc \
 	parallel.cc \
 	optimized_path.cc \
 	hex_dump.cc \
 	../types/string.cc \
+	iostream_adaptors.cc \
 
 LIBBASE_LINK :=	\
 	arch \