Skip to content

Commit

Permalink
quick fixes, probably didn't cover everything. Waiting for changelog
Browse files Browse the repository at this point in the history
  • Loading branch information
thatstoasty committed Mar 28, 2024
1 parent a7c1441 commit 05ec5f6
Show file tree
Hide file tree
Showing 15 changed files with 129 additions and 128 deletions.
12 changes: 6 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ All of these packages are partially implemented and do not support unicode chara
### Gojo

- `builtins`
- `Bytes` struct (backed by DynamicVector[Int8])
- `Bytes` struct (backed by List[Int8])
- `bufio`
- `Reader`: Buffered `io.Reader`
- `Scanner`: Scanner interface to read data via tokens.
Expand Down Expand Up @@ -67,7 +67,7 @@ fn test_bytes() raises:
bytes += String(" World").as_bytes()
test.assert_equal(str(bytes), "hellof World")

var bytes2 = DynamicVector[Int8]()
var bytes2 = List[Int8]()
bytes2.append(104)
bytes.extend(bytes2)
test.assert_equal(str(bytes), "hellof Worldh")
Expand All @@ -94,7 +94,7 @@ fn test_scan_words() raises:
var scanner = Scanner(r ^)
scanner.split = scan_words

var expected_results = DynamicVector[String]()
var expected_results = List[String]()
expected_results.append("Testing")
expected_results.append("this")
expected_results.append("string!")
Expand All @@ -116,7 +116,7 @@ fn test_scan_lines() raises:
# Create a scanner from the reader
var scanner = Scanner(r ^)

var expected_results = DynamicVector[String]()
var expected_results = List[String]()
expected_results.append("Testing")
expected_results.append("this")
expected_results.append("string!")
Expand All @@ -139,7 +139,7 @@ fn test_scan_bytes() raises:
var scanner = Scanner(r ^)
scanner.split = scan_bytes

var expected_results = DynamicVector[String]()
var expected_results = List[String]()
expected_results.append("a")
expected_results.append("b")
expected_results.append("c")
Expand All @@ -156,7 +156,7 @@ fn test_file_wrapper_scanner() raises:

# Create a scanner from the reader
var scanner = Scanner(file ^)
var expected_results = DynamicVector[String]()
var expected_results = List[String]()
expected_results.append("11111")
expected_results.append("22222")
expected_results.append("33333")
Expand Down
32 changes: 16 additions & 16 deletions external/csv/csv_table.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ alias simd_width_u8 = simdwidthof[DType.uint8]()

struct CsvTable:
var _inner_string: String
var _starts: DynamicVector[Int]
var _ends: DynamicVector[Int]
var _starts: List[Int]
var _ends: List[Int]
var column_count: Int

fn __init__(inout self, owned s: String, with_simd: Bool = True):
self._inner_string = s
self._starts = DynamicVector[Int](capacity=10)
self._ends = DynamicVector[Int](capacity=10)
self._starts = List[Int](capacity=10)
self._ends = List[Int](capacity=10)
self.column_count = -1
if with_simd:
self._simd_parse()
Expand All @@ -36,48 +36,48 @@ struct CsvTable:
var length = len(self._inner_string)
var offset = 0
var in_double_quotes = False
self._starts.push_back(offset)
self._starts.append(offset)
while offset < length:
var c = self._inner_string._buffer[offset]
if c == QUOTE:
in_double_quotes = not in_double_quotes
offset += 1
elif not in_double_quotes and c == COMMA:
self._ends.push_back(offset)
self._ends.append(offset)
offset += 1
self._starts.push_back(offset)
self._starts.append(offset)
elif not in_double_quotes and c == LF:
self._ends.push_back(offset)
self._ends.append(offset)
if self.column_count == -1:
self.column_count = len(self._ends)
offset += 1
self._starts.push_back(offset)
self._starts.append(offset)
elif (
not in_double_quotes
and c == CR
and length > offset + 1
and self._inner_string._buffer[offset + 1] == LF
):
self._ends.push_back(offset)
self._ends.append(offset)
if self.column_count == -1:
self.column_count = len(self._ends)
offset += 2
self._starts.push_back(offset)
self._starts.append(offset)
else:
offset += 1

if self._inner_string[length - 1] == "\n":
_ = self._starts.pop_back()
else:
self._ends.push_back(length)
self._ends.append(length)

@always_inline
fn _simd_parse(inout self):
var p = self._inner_string._as_ptr().bitcast[DType.uint8]()
var string_byte_length = len(self._inner_string)
var in_quotes = False
var last_chunk__ends_on_cr = False
self._starts.push_back(0)
self._starts.append(0)

@always_inline
@parameter
Expand Down Expand Up @@ -109,8 +109,8 @@ struct CsvTable:
rs_compensation = (lfs[index] & crs[index - 1]).to_int()
else:
rs_compensation = (lfs[index] & last_chunk__ends_on_cr).to_int()
self._ends.push_back(current_offset - rs_compensation)
self._starts.push_back(current_offset + 1)
self._ends.append(current_offset - rs_compensation)
self._starts.append(current_offset + 1)
if self.column_count == -1 and lfs[index]:
self.column_count = len(self._ends)
last_chunk__ends_on_cr = crs[simd_width - 1]
Expand All @@ -119,7 +119,7 @@ struct CsvTable:
if self._inner_string[string_byte_length - 1] == "\n":
_ = self._starts.pop_back()
else:
self._ends.push_back(string_byte_length)
self._ends.append(string_byte_length)

fn get(self, row: Int, column: Int) -> String:
if column >= self.column_count:
Expand Down
8 changes: 4 additions & 4 deletions external/csv/string_utils.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ fn vectorize_and_exit[
_ = workgroup_function[1](size - rest)


fn find_indices(s: String, c: String) -> DynamicVector[UInt64]:
fn find_indices(s: String, c: String) -> List[UInt64]:
var size = len(s)
var result = DynamicVector[UInt64]()
var result = List[UInt64]()
var char = Int8(ord(c))
var p = s._as_ptr()

Expand All @@ -71,7 +71,7 @@ fn find_indices(s: String, c: String) -> DynamicVector[UInt64]:
@parameter
if simd_width == 1:
if p.offset(offset).load() == char:
return result.push_back(offset)
return result.append(offset)
else:
var chunk = p.simd_load[simd_width](offset)
var occurrence = chunk == char
Expand Down Expand Up @@ -152,7 +152,7 @@ fn string_from_pointer(p: DTypePointer[DType.int8], length: Int) -> String:
return String(p, length)


fn print_v(v: DynamicVector[UInt64]):
fn print_v(v: List[UInt64]):
print_no_newline("(", len(v), ")", "[")
for i in range(len(v)):
print_no_newline(v[i], ",")
Expand Down
16 changes: 8 additions & 8 deletions external/libc.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@ alias char_pointer = AnyPointer[c_char]

@value
struct Str:
var vector: DynamicVector[c_char]
var vector: List[c_char]

fn __init__(inout self, string: String):
self.vector = DynamicVector[c_char](capacity=len(string) + 1)
self.vector = List[c_char](capacity=len(string) + 1)
for i in range(len(string)):
self.vector.push_back(ord(string[i]))
self.vector.push_back(0)
self.vector.append(ord(string[i]))
self.vector.append(0)

fn __init__(inout self, size: Int):
self.vector = DynamicVector[c_char]()
self.vector = List[c_char]()
self.vector.resize(size + 1, 0)

fn __len__(self) -> Int:
Expand Down Expand Up @@ -340,14 +340,14 @@ struct in_addr:
@value
@register_passable("trivial")
struct in6_addr:
var s6_addr: StaticTuple[16, c_char]
var s6_addr: StaticTuple[c_char, 16]


@value
@register_passable("trivial")
struct sockaddr:
var sa_family: sa_family_t
var sa_data: StaticTuple[14, c_char]
var sa_data: StaticTuple[c_char, 14]


@value
Expand All @@ -356,7 +356,7 @@ struct sockaddr_in:
var sin_family: sa_family_t
var sin_port: in_port_t
var sin_addr: in_addr
var sin_zero: StaticTuple[8, c_char]
var sin_zero: StaticTuple[c_char, 8]


@value
Expand Down
8 changes: 4 additions & 4 deletions gojo/bufio/bufio.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ struct Reader[R: io.Reader](
inout self,
delim: Int8,
inout frag: Bytes,
inout full_buffers: DynamicVector[Bytes],
inout full_buffers: List[Bytes],
inout total_len: Int,
) -> Optional[WrappedError]:
"""Reads until the first occurrence of delim in the input. It
Expand Down Expand Up @@ -491,7 +491,7 @@ struct Reader[R: io.Reader](
Returns:
The Bytes from the internal buffer.
"""
var full = DynamicVector[Bytes]()
var full = List[Bytes]()
var frag = Bytes(4096)
var n: Int = 0
var err = self.collect_fragments(delim, frag, full, n)
Expand Down Expand Up @@ -524,7 +524,7 @@ struct Reader[R: io.Reader](
Returns:
The String from the internal buffer.
"""
var full = DynamicVector[Bytes]()
var full = List[Bytes]()
var frag = Bytes()
var n: Int = 0
var err = self.collect_fragments(delim, frag, full, n)
Expand Down Expand Up @@ -680,7 +680,7 @@ struct Writer[W: io.Writer](
"""Returns the size of the underlying buffer in bytes."""
return len(self.buf)

fn reset[W: io.Writer](inout self, owned writer: W):
fn reset(inout self, owned writer: W):
"""Discards any unflushed buffered data, clears any error, and
resets b to write its output to w.
Calling reset on the zero value of [Writer] initializes the internal buffer
Expand Down
10 changes: 5 additions & 5 deletions gojo/bufio/scan.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ struct Scanner[R: io.Reader]():
if len(self.buf) >= self.max_token_size or len(self.buf) > int(
MAX_INT / 2
):
self.set_err(Err(Error(ERR_TOO_LONG)))
self.set_err(WrappedError(ERR_TOO_LONG))
return False

var new_size = len(self.buf) * 2
Expand Down Expand Up @@ -187,7 +187,7 @@ struct Scanner[R: io.Reader]():
error = result.get_error()
_ = copy(self.buf, sl, self.end)
if bytes_read < 0 or len(self.buf) - self.end < bytes_read:
self.set_err(Err(ERR_BAD_READ_COUNT))
self.set_err(WrappedError(ERR_BAD_READ_COUNT))
break

self.end += bytes_read
Expand All @@ -201,7 +201,7 @@ struct Scanner[R: io.Reader]():

loop += 1
if loop > MAX_CONSECUTIVE_EMPTY_READS:
self.set_err(Err(Error(io.ERR_NO_PROGRESS)))
self.set_err(WrappedError(io.ERR_NO_PROGRESS))
break

fn set_err(inout self, err: Err):
Expand All @@ -227,11 +227,11 @@ struct Scanner[R: io.Reader]():
True if the advance was legal, False otherwise.
"""
if n < 0:
self.set_err(Err(Error(ERR_NEGATIVE_ADVANCE)))
self.set_err(WrappedError(ERR_NEGATIVE_ADVANCE))
return False

if n > self.end - self.start:
self.set_err(Err(Error(ERR_ADVANCE_TOO_FAR)))
self.set_err(WrappedError(ERR_ADVANCE_TOO_FAR))
return False

self.start += n
Expand Down
24 changes: 12 additions & 12 deletions gojo/builtins/_bytes.mojo
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,24 @@ struct Bytes(Stringable, Sized, CollectionElement):
even some_bytes[7] = some_other_byte (the latter must be only one byte long).
"""

var _vector: DynamicVector[Int8]
var _vector: List[Int8]
var write_position: Int

fn __init__(inout self, size: Int = 0):
self.write_position = 0
if size != 0:
self._vector = DynamicVector[Int8](capacity=size)
self._vector = List[Int8](capacity=size)
for i in range(size):
self._vector.append(0)
else:
self._vector = DynamicVector[Int8]()
self._vector = List[Int8]()

fn __init__(inout self, owned vector: DynamicVector[Int8]):
fn __init__(inout self, owned vector: List[Int8]):
self.write_position = len(vector)
self._vector = vector

fn __init__(inout self, *strs: String):
self._vector = DynamicVector[Int8]()
self._vector = List[Int8]()
var total_length = 0
for string in strs:
self._vector.extend(string[].as_bytes())
Expand Down Expand Up @@ -70,7 +70,7 @@ struct Bytes(Stringable, Sized, CollectionElement):
# TODO: If no end was given, then it defaults to that large int.
# Accidentally including the 0 (null) characters will mess up strings due to null termination. __str__ expects the exact length of the string from self.write_position.
var end = limits.end
if limits.end == 9223372036854775807:
if limits.end == 2147483647:
end = self.size()
elif limits.end > self.size() + 1:
panic(
Expand Down Expand Up @@ -115,11 +115,11 @@ struct Bytes(Stringable, Sized, CollectionElement):
return not self.__eq__(other)

fn __add__(self, other: Self) -> Self:
var new_vector = DynamicVector[Int8](capacity=len(self) + len(other))
var new_vector = List[Int8](capacity=len(self) + len(other))
for i in range(len(self)):
new_vector.push_back(self[i])
new_vector.append(self[i])
for i in range(len(other)):
new_vector.push_back(other[i])
new_vector.append(other[i])
return Bytes(new_vector)

fn __iadd__(inout self: Self, other: Self):
Expand Down Expand Up @@ -170,7 +170,7 @@ struct Bytes(Stringable, Sized, CollectionElement):
"""
self += value

fn extend(inout self, value: DynamicVector[Int8]):
fn extend(inout self, value: List[Int8]):
"""Appends the values to the end of the Bytes.
Args:
Expand Down Expand Up @@ -232,7 +232,7 @@ struct Bytes(Stringable, Sized, CollectionElement):
bytes_copy.append(self._vector[i])
return bytes_copy

fn get_bytes(self) -> DynamicVector[Int8]:
fn get_bytes(self) -> List[Int8]:
"""
Returns a copy of the byte array of the string builder.
Expand All @@ -241,7 +241,7 @@ struct Bytes(Stringable, Sized, CollectionElement):
"""
return self.copy()._vector

fn get_null_terminated_bytes(self) -> DynamicVector[Int8]:
fn get_null_terminated_bytes(self) -> List[Int8]:
"""
Returns a copy of the byte array of the string builder with a null terminator.
Expand Down
Loading

0 comments on commit 05ec5f6

Please sign in to comment.