diff --git a/Cargo.lock b/Cargo.lock index adffcc315e..a8cfc86308 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -970,10 +970,14 @@ dependencies = [ name = "bench-vortex" version = "0.1.0" dependencies = [ + "arrow 51.0.0", "arrow-array 50.0.0", "arrow-array 51.0.0", + "arrow-csv 51.0.0", "arrow-select 51.0.0", + "bzip2", "criterion", + "csv", "itertools 0.12.1", "lance", "log", diff --git a/Cargo.toml b/Cargo.toml index 99e5ed64ff..048be67b71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,9 @@ thiserror = "1.0.58" uninit = "0.6.2" walkdir = "2.5.0" zigzag = "0.1.0" +bzip2 = "0.4.4" +csv = "1.3.0" +arrow-csv = "51.0.0" [workspace.lints.rust] warnings = "deny" diff --git a/bench-vortex/Cargo.toml b/bench-vortex/Cargo.toml index 2557eaebaa..d62d1d99b8 100644 --- a/bench-vortex/Cargo.toml +++ b/bench-vortex/Cargo.toml @@ -37,14 +37,19 @@ parquet = { workspace = true, features = [] } reqwest = { workspace = true } simplelog = { workspace = true } tokio = "1.0.1" +bzip2 = { workspace = true } +csv = { workspace = true } +arrow-csv = { workspace = true } +arrow = {workspace = true } [dev-dependencies] criterion = { workspace = true } + [[bench]] name = "compress_benchmark" harness = false [[bench]] name = "random_access" -harness = false \ No newline at end of file +harness = false diff --git a/bench-vortex/benches/compress_benchmark.rs b/bench-vortex/benches/compress_benchmark.rs index fcf00693a7..1c0e98e57e 100644 --- a/bench-vortex/benches/compress_benchmark.rs +++ b/bench-vortex/benches/compress_benchmark.rs @@ -1,8 +1,9 @@ -use bench_vortex::compress_taxi_data; +use bench_vortex::medicare_data::medicare_data_csv; use bench_vortex::taxi_data::taxi_data_parquet; +use bench_vortex::{compress_medicare_data, compress_taxi_data}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -fn vortex_compress(c: &mut Criterion) { +fn vortex_compress_taxi(c: &mut Criterion) { taxi_data_parquet(); let mut group = c.benchmark_group("end to end"); group.sample_size(10); @@ -10,5 +11,15 @@ fn vortex_compress(c: &mut Criterion) { group.finish() } -criterion_group!(benches, vortex_compress); +fn vortex_compress_medicare(c: &mut Criterion) { + medicare_data_csv(); + let mut group = c.benchmark_group("end to end"); + group.sample_size(10); + group.bench_function("compress", |b| { + b.iter(|| black_box(compress_medicare_data())) + }); + group.finish() +} + +criterion_group!(benches, vortex_compress_taxi, vortex_compress_medicare); criterion_main!(benches); diff --git a/bench-vortex/benches/random_access.rs b/bench-vortex/benches/random_access.rs index c9b28ee1c2..819758b6a4 100644 --- a/bench-vortex/benches/random_access.rs +++ b/bench-vortex/benches/random_access.rs @@ -1,3 +1,4 @@ +use bench_vortex::medicare_data::medicare_data_lance; use bench_vortex::reader::{take_lance, take_parquet, take_vortex}; use bench_vortex::taxi_data::{taxi_data_lance, taxi_data_parquet, taxi_data_vortex}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; @@ -22,6 +23,11 @@ fn random_access(c: &mut Criterion) { group.bench_function("lance", |b| { b.iter(|| black_box(take_lance(&taxi_lance, &indices))) }); + + let medicare_lance = medicare_data_lance(); + group.bench_function("lance", |b| { + b.iter(|| black_box(take_lance(&medicare_lance, &indices))) + }); } criterion_group!(benches, random_access); diff --git a/bench-vortex/src/bin/compress.rs b/bench-vortex/src/bin/compress.rs index 54689b42bd..134b60e113 100644 --- a/bench-vortex/src/bin/compress.rs +++ b/bench-vortex/src/bin/compress.rs @@ -2,20 +2,29 @@ use std::fs::File; use std::os::unix::prelude::MetadataExt; use std::path::PathBuf; -use bench_vortex::reader::{compress_parquet_to_vortex, open_vortex}; -use bench_vortex::setup_logger; +use bench_vortex::medicare_data::medicare_data_csv; +use bench_vortex::reader::{ + compress_csv_to_vortex, default_csv_format, open_vortex, rewrite_parquet_as_vortex, + write_csv_as_parquet, +}; use bench_vortex::taxi_data::taxi_data_parquet; +use bench_vortex::{data_path, setup_logger}; use log::LevelFilter; use vortex::array::Array; use vortex::formatter::display_tree; pub fn main() { setup_logger(LevelFilter::Debug); + compress_taxi(); + compress_medicare(); + write_medicare_as_parquet(); +} - let path: PathBuf = "taxi_data.vortex".into(); +fn compress_taxi() { + let path: PathBuf = data_path("taxi_data.vortex"); { let mut write = File::create(&path).unwrap(); - compress_parquet_to_vortex(&taxi_data_parquet(), &mut write).unwrap(); + rewrite_parquet_as_vortex(taxi_data_parquet(), &mut write).unwrap(); } let taxi_vortex = open_vortex(&path).unwrap(); @@ -27,3 +36,34 @@ pub fn main() { println!("Parquet size: {}, Vortex size: {}", pq_size, vx_size); println!("Compression ratio: {}", vx_size as f32 / pq_size as f32); } + +fn compress_medicare() { + let path: PathBuf = data_path("medicare.vortex"); + { + let mut write = File::create(&path).unwrap(); + let delimiter = u8::try_from('|').unwrap(); + compress_csv_to_vortex( + medicare_data_csv(), + default_csv_format().with_delimiter(delimiter), + &mut write, + ) + .unwrap(); + } + + let medicare_vortex = open_vortex(&path).unwrap(); + + let pq_size = medicare_data_csv().metadata().unwrap().size(); + let vx_size = medicare_vortex.nbytes(); + + println!("{}\n\n", display_tree(medicare_vortex.as_ref())); + println!("Csv size: {}, Vortex size: {}", pq_size, vx_size); + println!("Compression ratio: {}", vx_size as f32 / pq_size as f32); +} + +pub fn write_medicare_as_parquet() { + let path = data_path("medicare.parquet"); + let delimiter = u8::try_from('|').unwrap(); + let format = default_csv_format().with_delimiter(delimiter); + let file = File::create(path).unwrap(); + write_csv_as_parquet(medicare_data_csv(), format, file).unwrap(); +} diff --git a/bench-vortex/src/data_downloads.rs b/bench-vortex/src/data_downloads.rs index e0d22f7843..473aa38e7e 100644 --- a/bench-vortex/src/data_downloads.rs +++ b/bench-vortex/src/data_downloads.rs @@ -1,7 +1,9 @@ use std::fs::File; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use arrow_array::RecordBatchReader; +use bzip2::read::BzDecoder; use itertools::Itertools; use lance::dataset::WriteParams; use lance::Dataset; @@ -12,10 +14,11 @@ use vortex::array::chunked::ChunkedArray; use vortex::array::IntoArray; use vortex::arrow::FromArrowType; use vortex::serde::WriteCtx; +use vortex_error::VortexError; use vortex_schema::DType; -use crate::idempotent; -use crate::reader::{compress_parquet_to_vortex, BATCH_SIZE}; +use crate::reader::BATCH_SIZE; +use crate::{data_path, idempotent}; pub fn download_data(fname: &str, data_url: &str) -> PathBuf { idempotent(fname, |path| { @@ -44,12 +47,6 @@ pub fn parquet_to_lance(lance_fname: &Path, read: File) -> PathBuf { PathBuf::from(lance_fname) } -pub fn parquet_to_vortex(output_fname: &Path, data_to_compress: PathBuf) -> PathBuf { - let mut write = File::create(output_fname).unwrap(); - compress_parquet_to_vortex(&data_to_compress, &mut write).unwrap(); - output_fname.to_path_buf() -} - pub fn data_vortex_uncompressed(fname_out: &str, downloaded_data: PathBuf) -> PathBuf { idempotent(fname_out, |path| { let taxi_pq = File::open(downloaded_data).unwrap(); @@ -73,3 +70,18 @@ pub fn data_vortex_uncompressed(fname_out: &str, downloaded_data: PathBuf) -> Pa }) .unwrap() } + +pub fn decompress_bz2(input_path: &str, output_path: &str) -> PathBuf { + idempotent(output_path, |path| { + let input_file = File::open(data_path(input_path)).unwrap(); + let mut decoder = BzDecoder::new(input_file); + + let mut buffer = Vec::new(); + decoder.read_to_end(&mut buffer).unwrap(); + + let mut output_file = File::create(path).unwrap(); + output_file.write_all(&buffer).unwrap(); + Ok::(data_path(output_path)) + }) + .unwrap() +} diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index 542d5925c8..6b4a0af74a 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -1,8 +1,11 @@ use std::fs::{create_dir_all, File}; +use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::Arc; +use arrow::datatypes::SchemaRef; use arrow_array::RecordBatchReader; +use arrow_csv::reader::Format; use itertools::Itertools; use log::{info, warn, LevelFilter}; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; @@ -24,9 +27,12 @@ use vortex_ree::REEEncoding; use vortex_roaring::RoaringBoolEncoding; use vortex_schema::DType; +use crate::medicare_data::medicare_data_csv; +use crate::reader::BATCH_SIZE; use crate::taxi_data::taxi_data_parquet; mod data_downloads; +pub mod medicare_data; pub mod reader; pub mod taxi_data; @@ -92,7 +98,7 @@ pub fn compress_taxi_data() -> ArrayRef { let reader = builder .with_projection(_mask) //.with_projection(no_datetime_mask) - .with_batch_size(65_536) + .with_batch_size(BATCH_SIZE) // .with_batch_size(5_000_000) // .with_limit(100_000) .build() @@ -100,11 +106,9 @@ pub fn compress_taxi_data() -> ArrayRef { let ctx = compress_ctx(); let schema = reader.schema(); - let mut uncompressed_size = 0; + let mut uncompressed_size: usize = 0; let chunks = reader .into_iter() - //.skip(39) - //.take(1) .map(|batch_result| batch_result.unwrap()) .map(|batch| batch.into_array()) .map(|array| { @@ -113,6 +117,38 @@ pub fn compress_taxi_data() -> ArrayRef { }) .collect_vec(); + chunks_to_array(schema, uncompressed_size, chunks) +} + +pub fn compress_medicare_data() -> ArrayRef { + let csv_file = File::open(medicare_data_csv()).unwrap(); + let reader = BufReader::new(csv_file.try_clone().unwrap()); + + let (schema, _) = Format::default() + .with_delimiter(u8::try_from('|').unwrap()) + .infer_schema(&mut csv_file.try_clone().unwrap(), None) + .unwrap(); + + let csv_reader = arrow::csv::ReaderBuilder::new(Arc::new(schema.clone())) + .with_batch_size(BATCH_SIZE * 10) + .build(reader) + .unwrap(); + + let ctx = compress_ctx(); + let mut uncompressed_size: usize = 0; + let chunks = csv_reader + .into_iter() + .map(|batch_result| batch_result.unwrap()) + .map(|batch| batch.into_array()) + .map(|array| { + uncompressed_size += array.nbytes(); + ctx.clone().compress(&array, None).unwrap() + }) + .collect_vec(); + chunks_to_array(SchemaRef::new(schema), uncompressed_size, chunks) +} + +fn chunks_to_array(schema: SchemaRef, uncompressed_size: usize, chunks: Vec) -> ArrayRef { let dtype = DType::from_arrow(schema.clone()); let compressed = ChunkedArray::new(chunks.clone(), dtype).into_array(); diff --git a/bench-vortex/src/medicare_data.rs b/bench-vortex/src/medicare_data.rs new file mode 100644 index 0000000000..f751a6cec2 --- /dev/null +++ b/bench-vortex/src/medicare_data.rs @@ -0,0 +1,87 @@ +use std::fs::File; +use std::io::BufReader; +use std::path::PathBuf; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use arrow_csv::reader::Format; +use itertools::Itertools; +use vortex::array::chunked::ChunkedArray; +use vortex::array::IntoArray; +use vortex::arrow::FromArrowType; +use vortex::serde::WriteCtx; +use vortex_error::VortexError; +use vortex_schema::DType; + +use crate::data_downloads::{decompress_bz2, download_data, parquet_to_lance}; +use crate::idempotent; +use crate::reader::{compress_csv_to_vortex, default_csv_format, write_csv_as_parquet}; + +pub fn medicare_data_csv() -> PathBuf { + let fname = "Medicare1_1.csv.bz2"; + download_data( + fname, + "http://www.cwi.nl/~boncz/PublicBIbenchmark/Medicare1/Medicare1_1.csv.bz2", + ); + decompress_bz2(fname, "Medicare1_1.csv") +} + +pub fn medicare_data_lance() -> PathBuf { + let taxi_data = File::open(medicare_data_parquet()).unwrap(); + idempotent("medicare.lance", |path| { + Ok::(parquet_to_lance(path, taxi_data)) + }) + .unwrap() +} + +pub fn medicare_data_vortex_uncompressed() -> PathBuf { + idempotent("medicare-uncompressed.vortex", |path| { + let csv_file = File::open(medicare_data_csv()).unwrap(); + let reader = BufReader::new(csv_file.try_clone().unwrap()); + + let (schema, _) = Format::default() + .infer_schema(&mut csv_file.try_clone().unwrap(), None) + .unwrap(); + + let csv_reader = arrow::csv::ReaderBuilder::new(Arc::new(schema.clone())) + .with_batch_size(crate::reader::BATCH_SIZE) + .build(reader)?; + + let dtype = DType::from_arrow(SchemaRef::new(schema.clone())); + + let chunks = csv_reader + .map(|batch_result| batch_result.unwrap()) + .map(|record_batch| record_batch.into_array()) + .collect_vec(); + let chunked = ChunkedArray::new(chunks, dtype.clone()); + + let mut write = File::create(path).unwrap(); + let mut write_ctx = WriteCtx::new(&mut write); + write_ctx.dtype(&dtype)?; + write_ctx.write(&chunked) + }) + .unwrap() +} + +pub fn medicare_data_vortex() -> PathBuf { + idempotent("medicare.vortex", |path| { + let mut write = File::create(path).unwrap(); + let delimiter = u8::try_from('|').unwrap(); + compress_csv_to_vortex( + medicare_data_csv(), + default_csv_format().with_delimiter(delimiter), + &mut write, + ) + }) + .unwrap() +} + +pub fn medicare_data_parquet() -> PathBuf { + idempotent("medicare.parquet", |path| { + let delimiter = u8::try_from('|').unwrap(); + let format = default_csv_format().with_delimiter(delimiter); + let file = File::create(path).unwrap(); + write_csv_as_parquet(medicare_data_csv(), format, file) + }) + .unwrap() +} diff --git a/bench-vortex/src/reader.rs b/bench-vortex/src/reader.rs index e401e0c8ad..dfa29afdd1 100644 --- a/bench-vortex/src/reader.rs +++ b/bench-vortex/src/reader.rs @@ -1,20 +1,26 @@ use std::collections::HashMap; use std::fs::File; -use std::io::Write; -use std::path::Path; +use std::io::{BufReader, Write}; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use arrow::csv; +use arrow::datatypes::SchemaRef; use arrow_array::types::Int64Type; use arrow_array::{ ArrayRef as ArrowArrayRef, PrimitiveArray as ArrowPrimitiveArray, RecordBatch, RecordBatchReader, }; +use arrow_csv::reader::Format; use arrow_select::concat::concat_batches; use arrow_select::take::take_record_batch; use itertools::Itertools; use lance::Dataset; use lance_arrow_array::RecordBatch as LanceRecordBatch; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::arrow::ArrowWriter; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; use tokio::runtime::Runtime; use vortex::array::chunked::ChunkedArray; use vortex::array::primitive::PrimitiveArray; @@ -24,12 +30,14 @@ use vortex::compute::flatten::flatten; use vortex::compute::take::take; use vortex::ptype::PType; use vortex::serde::{ReadCtx, WriteCtx}; -use vortex_error::VortexResult; +use vortex_error::{VortexError, VortexResult}; use vortex_schema::DType; use crate::compress_ctx; pub const BATCH_SIZE: usize = 65_536; +const CSV_SCHEMA_SAMPLE_ROWS: usize = 100; +const DEFAULT_DELIMITER: u8 = b','; pub fn open_vortex(path: &Path) -> VortexResult { let mut file = File::open(path)?; @@ -39,10 +47,19 @@ pub fn open_vortex(path: &Path) -> VortexResult { read_ctx.with_schema(&dtype).read() } -pub fn compress_parquet_to_vortex( - parquet_path: &Path, +pub fn rewrite_parquet_as_vortex( + parquet_path: PathBuf, write: &mut W, ) -> VortexResult<()> { + let (dtype, chunked) = compress_parquet_to_vortex(parquet_path.as_path())?; + + let mut write_ctx = WriteCtx::new(write); + write_ctx.dtype(&dtype).unwrap(); + write_ctx.write(&chunked).unwrap(); + Ok(()) +} + +fn compress_parquet_to_vortex(parquet_path: &Path) -> Result<(DType, ChunkedArray), VortexError> { let taxi_pq = File::open(parquet_path)?; let builder = ParquetRecordBatchReaderBuilder::try_new(taxi_pq)?; @@ -60,6 +77,46 @@ pub fn compress_parquet_to_vortex( }) .collect_vec(); let chunked = ChunkedArray::new(chunks, dtype.clone()); + Ok((dtype, chunked)) +} + +pub fn default_csv_format() -> Format { + Format::default() + .with_delimiter(DEFAULT_DELIMITER) + .with_header(false) + .with_null_regex("null".parse().unwrap()) +} + +pub fn compress_csv_to_vortex( + csv_path: PathBuf, + format: Format, + write: &mut W, +) -> VortexResult<()> { + let csv_file_for_schema = File::open(csv_path.clone()).unwrap(); + + // Infer the schema of the CSV file + let (schema, _) = format.infer_schema( + csv_file_for_schema.try_clone().unwrap(), + Some(CSV_SCHEMA_SAMPLE_ROWS), + )?; + let csv_file_for_read = File::open(csv_path.clone()).unwrap(); + let file_reader = BufReader::new(csv_file_for_read.try_clone().unwrap()); + + let csv_reader = csv::ReaderBuilder::new(SchemaRef::new(schema.clone())) + .with_format(format.clone()) + .with_batch_size(BATCH_SIZE) + .build(file_reader)?; + let dtype = DType::from_arrow(SchemaRef::from(schema.clone())); + let ctx = compress_ctx(); + + let chunks = csv_reader + .map(|batch_result| batch_result.unwrap()) + .map(|record_batch| { + let vortex_array = record_batch.into_array(); + ctx.compress(&vortex_array, None).unwrap() + }) + .collect_vec(); + let chunked = ChunkedArray::new(chunks, dtype.clone()); let mut write_ctx = WriteCtx::new(write); write_ctx.dtype(&dtype).unwrap(); @@ -67,6 +124,42 @@ pub fn compress_parquet_to_vortex( Ok(()) } +pub fn write_csv_as_parquet( + csv_path: PathBuf, + format: Format, + write: W, +) -> VortexResult<()> { + let file_handle_for_schema_inference = File::open(csv_path.clone()).unwrap(); + + // Infer the schema of the CSV file + let schema_inference_reader = + BufReader::new(file_handle_for_schema_inference.try_clone().unwrap()); + let (schema, _) = format.infer_schema(schema_inference_reader, Some(CSV_SCHEMA_SAMPLE_ROWS))?; + let file_handle_for_read = File::open(csv_path.clone()).unwrap(); + + let file_reader = BufReader::new(file_handle_for_read.try_clone().unwrap()); + let csv_reader = csv::ReaderBuilder::new(SchemaRef::new(schema.clone())) + .with_format(format) + .with_batch_size(BATCH_SIZE) + .build(file_reader)?; + // WriterProperties can be used to set Parquet file options + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let mut writer = ArrowWriter::try_new(write, SchemaRef::from(schema), Some(props)).unwrap(); + + // Write CSV data to Parquet + for maybe_batch in csv_reader { + let record_batch = maybe_batch?; + writer.write(&record_batch)?; + } + + // Finalize the Parquet writer + writer.close()?; + Ok(()) +} + pub fn take_vortex(path: &Path, indices: &[u64]) -> VortexResult { let array = open_vortex(path)?; let taken = take(&array, &PrimitiveArray::from(indices.to_vec()))?; diff --git a/bench-vortex/src/taxi_data.rs b/bench-vortex/src/taxi_data.rs index ca646b2a99..d2070ff83e 100644 --- a/bench-vortex/src/taxi_data.rs +++ b/bench-vortex/src/taxi_data.rs @@ -3,16 +3,15 @@ use std::path::PathBuf; use vortex_error::VortexError; -use crate::data_downloads::{ - data_vortex_uncompressed, download_data, parquet_to_lance, parquet_to_vortex, -}; +use crate::data_downloads::{data_vortex_uncompressed, download_data, parquet_to_lance}; use crate::idempotent; +use crate::reader::rewrite_parquet_as_vortex; fn download_taxi_data() -> PathBuf { - let taxi_parquet_fname = "yellow-tripdata-2023-11.parquet"; + let taxi_parquet_fpath = "yellow-tripdata-2023-11.parquet"; let taxi_data_url = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-11.parquet"; - download_data(taxi_parquet_fname, taxi_data_url) + download_data(taxi_parquet_fpath, taxi_data_url) } pub fn taxi_data_parquet() -> PathBuf { @@ -33,7 +32,9 @@ pub fn taxi_data_vortex_uncompressed() -> PathBuf { pub fn taxi_data_vortex() -> PathBuf { idempotent("taxi.vortex", |output_fname| { - Ok::(parquet_to_vortex(output_fname, taxi_data_parquet())) + let mut output_file = File::create(output_fname)?; + rewrite_parquet_as_vortex(taxi_data_parquet(), &mut output_file)?; + Ok::(output_fname.to_path_buf()) }) .unwrap() } diff --git a/requirements-dev.lock b/requirements-dev.lock index 6e964b7647..69b5191d9f 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -7,8 +7,8 @@ # all-features: false # with-sources: false --e file:pyvortex -e file:. +-e file:pyvortex babel==2.14.0 # via mkdocs-material bracex==2.4 @@ -25,9 +25,9 @@ ghp-import==2.1.0 # via mkdocs idna==3.6 # via requests -importlib-metadata==7.0.1 +importlib-metadata==7.1.0 # via mike -importlib-resources==6.1.2 +importlib-resources==6.4.0 # via mike iniconfig==2.0.0 # via pytest @@ -35,14 +35,14 @@ jinja2==3.1.3 # via mike # via mkdocs # via mkdocs-material -markdown==3.5.2 +markdown==3.6 # via mkdocs # via mkdocs-material # via pymdown-extensions markupsafe==2.1.5 # via jinja2 # via mkdocs -maturin==1.4.0 +maturin==1.5.1 mergedeep==1.3.4 # via mkdocs mike==2.0.0 @@ -50,36 +50,37 @@ mkdocs==1.5.3 # via mike # via mkdocs-include-markdown-plugin # via mkdocs-material -mkdocs-include-markdown-plugin==6.0.4 -mkdocs-material==9.5.12 +mkdocs-include-markdown-plugin==6.0.5 +mkdocs-material==9.5.17 mkdocs-material-extensions==1.3.1 # via mkdocs-material numpy==1.26.4 # via pyarrow -packaging==23.2 +packaging==24.0 # via mkdocs # via pytest paginate==0.5.6 # via mkdocs-material pathspec==0.12.1 # via mkdocs +pip==24.0 platformdirs==4.2.0 # via mkdocs pluggy==1.4.0 # via pytest py-cpuinfo==9.0.0 # via pytest-benchmark -pyarrow==15.0.0 +pyarrow==15.0.2 pygments==2.17.2 # via mkdocs-material -pymdown-extensions==10.7 +pymdown-extensions==10.7.1 # via mkdocs-material -pyparsing==3.1.1 +pyparsing==3.1.2 # via mike -pytest==7.4.0 +pytest==8.1.1 # via pytest-benchmark pytest-benchmark==4.0.0 -python-dateutil==2.9.0 +python-dateutil==2.9.0.post0 # via ghp-import pyyaml==6.0.1 # via mike @@ -92,7 +93,7 @@ regex==2023.12.25 # via mkdocs-material requests==2.31.0 # via mkdocs-material -ruff==0.2.2 +ruff==0.3.5 six==1.16.0 # via python-dateutil urllib3==2.2.1 @@ -103,6 +104,5 @@ watchdog==4.0.0 # via mkdocs wcmatch==8.5.1 # via mkdocs-include-markdown-plugin -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata -pip==24.0 diff --git a/requirements.lock b/requirements.lock index 8f59ff5e9c..bf905daba0 100644 --- a/requirements.lock +++ b/requirements.lock @@ -7,5 +7,5 @@ # all-features: false # with-sources: false --e file:pyvortex -e file:. +-e file:pyvortex