Skip to content

Commit

Permalink
Backport: Fix serialization of large integers (#5038) (#5042) to `48.…
Browse files Browse the repository at this point in the history
…0.0_maintenance` (#5059)

* Fix serialization of large integers (#5038) (#5042)

* fmt

---------

Co-authored-by: Raphael Taylor-Davies <[email protected]>
  • Loading branch information
alamb and tustvold authored Nov 9, 2023
1 parent f91a096 commit a5027e8
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 3 deletions.
30 changes: 30 additions & 0 deletions arrow-json/src/reader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2239,4 +2239,34 @@ mod tests {
let values = b.column(0).as_primitive::<Int32Type>().values();
assert_eq!(values, &[1, 2, 3, 4]);
}

#[test]
fn test_serde_large_numbers() {
let field = Field::new("int", DataType::Int64, true);
let mut decoder = ReaderBuilder::new_with_field(field)
.build_decoder()
.unwrap();

decoder.serialize(&[1699148028689_u64, 2, 3, 4]).unwrap();
let b = decoder.flush().unwrap().unwrap();
let values = b.column(0).as_primitive::<Int64Type>().values();
assert_eq!(values, &[1699148028689, 2, 3, 4]);

let field = Field::new(
"int",
DataType::Timestamp(TimeUnit::Microsecond, None),
true,
);
let mut decoder = ReaderBuilder::new_with_field(field)
.build_decoder()
.unwrap();

decoder.serialize(&[1699148028689_u64, 2, 3, 4]).unwrap();
let b = decoder.flush().unwrap().unwrap();
let values = b
.column(0)
.as_primitive::<TimestampMicrosecondType>()
.values();
assert_eq!(values, &[1699148028689, 2, 3, 4]);
}
}
2 changes: 1 addition & 1 deletion arrow-json/src/reader/primitive_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ where
},
TapeElement::I64(high) => match tape.get(p + 1) {
TapeElement::I32(low) => {
let v = (high as i64) << 32 | low as i64;
let v = (high as i64) << 32 | (low as u32) as i64;
let value = NumCast::from(v).ok_or_else(|| {
ArrowError::JsonError(format!("failed to parse {v} as {d}",))
})?;
Expand Down
2 changes: 1 addition & 1 deletion arrow-json/src/reader/tape.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ impl<'a> Tape<'a> {
TapeElement::Null => out.push_str("null"),
TapeElement::I64(high) => match self.get(idx + 1) {
TapeElement::I32(low) => {
let val = (high as i64) << 32 | low as i64;
let val = (high as i64) << 32 | (low as u32) as i64;
let _ = write!(out, "{val}");
return idx + 2;
}
Expand Down
2 changes: 1 addition & 1 deletion arrow-json/src/reader/timestamp_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ where
TapeElement::I32(v) => builder.append_value(v as i64),
TapeElement::I64(high) => match tape.get(p + 1) {
TapeElement::I32(low) => {
builder.append_value((high as i64) << 32 | low as i64)
builder.append_value((high as i64) << 32 | (low as u32) as i64)
}
_ => unreachable!(),
},
Expand Down

0 comments on commit a5027e8

Please sign in to comment.