Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(fuzzer): add supporting map functions #11536

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+51 −13
Open

refactor(fuzzer): add supporting map functions #11536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 36 additions & 13 deletions velox/vector/fuzzer/VectorFuzzer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,16 +537,7 @@ VectorPtr VectorFuzzer::fuzzFlat(const TypePtr& type, vector_size_t size) {
// Do not initialize keys and values inline in the fuzzMap call as C++ does
// not specify the order they'll be called in, leading to inconsistent
// results across platforms.
const auto& keyType = type->asMap().keyType();
const auto& valueType = type->asMap().valueType();
auto length = getElementsVectorLength(opts_, size);

auto keys = opts_.normalizeMapKeys || !opts_.containerHasNulls
? fuzzFlatNotNull(keyType, length)
: fuzzFlat(keyType, length);
auto values = opts_.containerHasNulls ? fuzzFlat(valueType, length)
: fuzzFlatNotNull(valueType, length);
return fuzzMap(keys, values, size);
return fuzzMap(type->asMap().keyType(), type->asMap().valueType(), size);
}
// Rows.
else if (type->isRow()) {
Expand All @@ -568,6 +559,24 @@ VectorPtr VectorFuzzer::fuzzFlat(const TypePtr& type, vector_size_t size) {
}
}

VectorPtr VectorFuzzer::fuzzKeys(const TypePtr& keyType, vector_size_t size) {
auto length = getElementsVectorLength(opts_, size);
return opts_.normalizeMapKeys || !opts_.containerHasNulls
? fuzzFlatNotNull(keyType, length)
: fuzzFlat(keyType, length);
}

VectorPtr VectorFuzzer::fuzzMap(
const TypePtr& keyType,
const TypePtr& valueType,
vector_size_t size) {
auto keys = fuzzKeys(keyType, size);
auto length = getElementsVectorLength(opts_, size);
auto values = opts_.containerHasNulls ? fuzzFlat(valueType, length)
: fuzzFlatNotNull(valueType, length);
return fuzzMap(keys, values, size);
}

VectorPtr VectorFuzzer::fuzzFlatPrimitive(
const TypePtr& type,
vector_size_t size) {
Expand Down Expand Up @@ -912,6 +921,10 @@ TypePtr VectorFuzzer::randType(
return velox::randType(rng_, scalarTypes, maxDepth);
}

TypePtr VectorFuzzer::randMapType(int maxDepth) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this method used?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. This function is used in my follow up diff D65926722 for controlling the number of maps in my row vector

return velox::randMapType(rng_, defaultScalarTypes(), maxDepth);
}

RowTypePtr VectorFuzzer::randRowType(int maxDepth) {
return velox::randRowType(rng_, maxDepth);
}
Expand All @@ -922,6 +935,10 @@ RowTypePtr VectorFuzzer::randRowType(
return velox::randRowType(rng_, scalarTypes, maxDepth);
}

size_t VectorFuzzer::randInRange(size_t min, size_t max) {
return rand(rng_, min, max);
}

VectorPtr VectorFuzzer::wrapInLazyVector(VectorPtr baseVector) {
if (hasNestedDictionaryLayers(baseVector)) {
auto indices = baseVector->wrapInfo();
Expand Down Expand Up @@ -1139,16 +1156,22 @@ TypePtr randType(
}
switch (rand<uint32_t>(rng) % 3) {
case 0:
return MAP(
randType(rng, scalarTypes, 0),
randType(rng, scalarTypes, maxDepth - 1));
return randMapType(rng, scalarTypes, maxDepth);
case 1:
return ARRAY(randType(rng, scalarTypes, maxDepth - 1));
default:
return randRowType(rng, scalarTypes, maxDepth - 1);
}
}

TypePtr randMapType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
int maxDepth) {
return MAP(
randType(rng, scalarTypes, 0), randType(rng, scalarTypes, maxDepth - 1));
}

TypePtr randOrderableType(FuzzerGenerator& rng, int maxDepth) {
return randOrderableType(rng, defaultScalarTypes(), maxDepth);
}
Expand Down
15 changes: 15 additions & 0 deletions velox/vector/fuzzer/VectorFuzzer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,12 @@ class VectorFuzzer {
VectorPtr fuzzFlatNotNull(const TypePtr& type);
VectorPtr fuzzFlatNotNull(const TypePtr& type, vector_size_t size);

/// Returns a map vector with randomized values and nulls.Returns a vector
/// containing `opts_.vectorSize` or `size` elements.
VectorPtr
fuzzMap(const TypePtr& keyType, const TypePtr& valueType, vector_size_t size);
VectorPtr fuzzKeys(const TypePtr& keyType, vector_size_t size);

/// Returns a random constant vector (which could be a null constant). Returns
/// a vector with size set to `opts_.vectorSize` or 'size'.
VectorPtr fuzzConstant(const TypePtr& type);
Expand Down Expand Up @@ -285,6 +291,10 @@ class VectorFuzzer {
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

size_t randInRange(size_t min, size_t max);

TypePtr randMapType(int maxDepth = 5);

/// Generates short decimal TypePtr with random precision and scale.
inline TypePtr randShortDecimalType() {
auto [precision, scale] =
Expand Down Expand Up @@ -406,6 +416,11 @@ TypePtr randType(
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

TypePtr randMapType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

/// Same as the function above, but only generate orderable types.
/// MAP types are not generated as they are not orderable.
TypePtr randOrderableType(FuzzerGenerator& rng, int maxDepth = 5);
Expand Down