Skip to content

Commit

Permalink
test(fuzzer): add supporting map functions (#11536)
Browse files Browse the repository at this point in the history
Summary:

This diff adds some supporting functions for my diff for new benchmark tests for map_concat.

Reviewed By: darrenfu

Differential Revision: D65755011
  • Loading branch information
wuray22 authored and facebook-github-bot committed Nov 24, 2024
1 parent 78d761b commit 5e20018
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 15 deletions.
61 changes: 46 additions & 15 deletions velox/vector/fuzzer/VectorFuzzer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -537,16 +537,7 @@ VectorPtr VectorFuzzer::fuzzFlat(const TypePtr& type, vector_size_t size) {
// Do not initialize keys and values inline in the fuzzMap call as C++ does
// not specify the order they'll be called in, leading to inconsistent
// results across platforms.
const auto& keyType = type->asMap().keyType();
const auto& valueType = type->asMap().valueType();
auto length = getElementsVectorLength(opts_, size);

auto keys = opts_.normalizeMapKeys || !opts_.containerHasNulls
? fuzzFlatNotNull(keyType, length)
: fuzzFlat(keyType, length);
auto values = opts_.containerHasNulls ? fuzzFlat(valueType, length)
: fuzzFlatNotNull(valueType, length);
return fuzzMap(keys, values, size);
return fuzzMap(type->asMap().keyType(), type->asMap().valueType(), size);
}
// Rows.
else if (type->isRow()) {
Expand All @@ -568,6 +559,24 @@ VectorPtr VectorFuzzer::fuzzFlat(const TypePtr& type, vector_size_t size) {
}
}

VectorPtr VectorFuzzer::fuzzKeys(const TypePtr& keyType, vector_size_t size) {
auto length = getElementsVectorLength(opts_, size);
return opts_.normalizeMapKeys || !opts_.containerHasNulls
? fuzzFlatNotNull(keyType, length)
: fuzzFlat(keyType, length);
}

VectorPtr VectorFuzzer::fuzzMap(
const TypePtr& keyType,
const TypePtr& valueType,
vector_size_t size) {
auto keys = fuzzKeys(keyType, size);
auto length = getElementsVectorLength(opts_, size);
auto values = opts_.containerHasNulls ? fuzzFlat(valueType, length)
: fuzzFlatNotNull(valueType, length);
return fuzzMap(keys, values, size);
}

VectorPtr VectorFuzzer::fuzzFlatPrimitive(
const TypePtr& type,
vector_size_t size) {
Expand Down Expand Up @@ -912,6 +921,10 @@ TypePtr VectorFuzzer::randType(
return velox::randType(rng_, scalarTypes, maxDepth);
}

TypePtr VectorFuzzer::randMapType(int maxDepth) {
return velox::randMapType(rng_, defaultScalarTypes(), maxDepth);
}

RowTypePtr VectorFuzzer::randRowType(int maxDepth) {
return velox::randRowType(rng_, maxDepth);
}
Expand All @@ -922,6 +935,10 @@ RowTypePtr VectorFuzzer::randRowType(
return velox::randRowType(rng_, scalarTypes, maxDepth);
}

size_t VectorFuzzer::randInRange(size_t min, size_t max) {
return rand(rng_, min, max);
}

VectorPtr VectorFuzzer::wrapInLazyVector(VectorPtr baseVector) {
if (hasNestedDictionaryLayers(baseVector)) {
auto indices = baseVector->wrapInfo();
Expand Down Expand Up @@ -1139,16 +1156,22 @@ TypePtr randType(
}
switch (rand<uint32_t>(rng) % 3) {
case 0:
return MAP(
randType(rng, scalarTypes, 0),
randType(rng, scalarTypes, maxDepth - 1));
return randMapType(rng, scalarTypes, maxDepth);
case 1:
return ARRAY(randType(rng, scalarTypes, maxDepth - 1));
default:
return randRowType(rng, scalarTypes, maxDepth - 1);
}
}

TypePtr randMapType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
int maxDepth) {
return MAP(
randType(rng, scalarTypes, 0), randType(rng, scalarTypes, maxDepth - 1));
}

TypePtr randOrderableType(FuzzerGenerator& rng, int maxDepth) {
return randOrderableType(rng, defaultScalarTypes(), maxDepth);
}
Expand Down Expand Up @@ -1185,10 +1208,18 @@ RowTypePtr randRowType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
int maxDepth) {
int numFields = 1 + rand<uint32_t>(rng) % 7;
size_t numFields = 1 + rand<uint32_t>(rng) % 7;
return randRowType(rng, scalarTypes, numFields, maxDepth);
}

RowTypePtr randRowType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
size_t numFields,
int maxDepth) {
std::vector<std::string> names;
std::vector<TypePtr> fields;
for (int i = 0; i < numFields; ++i) {
for (size_t i = 0; i < numFields; ++i) {
names.push_back(fmt::format("f{}", i));
fields.push_back(randType(rng, scalarTypes, maxDepth));
}
Expand Down
21 changes: 21 additions & 0 deletions velox/vector/fuzzer/VectorFuzzer.h
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,12 @@ class VectorFuzzer {
VectorPtr fuzzFlatNotNull(const TypePtr& type);
VectorPtr fuzzFlatNotNull(const TypePtr& type, vector_size_t size);

/// Returns a map vector with randomized values and nulls.Returns a vector
/// containing `opts_.vectorSize` or `size` elements.
VectorPtr
fuzzMap(const TypePtr& keyType, const TypePtr& valueType, vector_size_t size);
VectorPtr fuzzKeys(const TypePtr& keyType, vector_size_t size);

/// Returns a random constant vector (which could be a null constant). Returns
/// a vector with size set to `opts_.vectorSize` or 'size'.
VectorPtr fuzzConstant(const TypePtr& type);
Expand Down Expand Up @@ -285,6 +291,10 @@ class VectorFuzzer {
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

size_t randInRange(size_t min, size_t max);

TypePtr randMapType(int maxDepth = 5);

/// Generates short decimal TypePtr with random precision and scale.
inline TypePtr randShortDecimalType() {
auto [precision, scale] =
Expand Down Expand Up @@ -406,6 +416,11 @@ TypePtr randType(
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

TypePtr randMapType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

/// Same as the function above, but only generate orderable types.
/// MAP types are not generated as they are not orderable.
TypePtr randOrderableType(FuzzerGenerator& rng, int maxDepth = 5);
Expand All @@ -423,6 +438,12 @@ RowTypePtr randRowType(
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

RowTypePtr randRowType(
FuzzerGenerator& rng,
const std::vector<TypePtr>& scalarTypes,
size_t numFields,
int maxDepth = 5);

/// Default set of scalar types to be chosen from when generating random types.
const std::vector<TypePtr>& defaultScalarTypes();

Expand Down

0 comments on commit 5e20018

Please sign in to comment.