venv/lib/python3.11/site-packages/pyarrow/include/parquet/test_util.h

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// This module defines an abstract interface for iterating through pages in a
// Parquet column chunk within a row group. It could be extended in the future
// to iterate through all data pages in all chunks in a file.

#pragma once

#include <algorithm>
#include <limits>
#include <memory>
#include <random>
#include <string>
#include <utility>
#include <vector>

#include <gtest/gtest.h>

#include "arrow/io/memory.h"
#include "arrow/testing/util.h"
#include "arrow/util/float16.h"

#include "parquet/column_page.h"
#include "parquet/column_reader.h"
#include "parquet/column_writer.h"
#include "parquet/encoding.h"
#include "parquet/platform.h"

// https://github.com/google/googletest/pull/2904 might not be available
// in our version of gtest/gmock
#define EXPECT_THROW_THAT(callable, ex_type, property)   \
  EXPECT_THROW(                                          \
      try { (callable)(); } catch (const ex_type& err) { \
        EXPECT_THAT(err, (property));                    \
        throw;                                           \
      },                                                 \
      ex_type)

namespace parquet {

static constexpr int FLBA_LENGTH = 12;

inline bool operator==(const FixedLenByteArray& a, const FixedLenByteArray& b) {
  return 0 == memcmp(a.ptr, b.ptr, FLBA_LENGTH);
}

namespace test {

typedef ::testing::Types<BooleanType, Int32Type, Int64Type, Int96Type, FloatType,
                         DoubleType, ByteArrayType, FLBAType>
    ParquetTypes;

class ParquetTestException : public parquet::ParquetException {
  using ParquetException::ParquetException;
};

const char* get_data_dir();
std::string get_bad_data_dir();

std::string get_data_file(const std::string& filename, bool is_good = true);

template <typename T>
static inline void assert_vector_equal(const std::vector<T>& left,
                                       const std::vector<T>& right) {
  ASSERT_EQ(left.size(), right.size());

  for (size_t i = 0; i < left.size(); ++i) {
    ASSERT_EQ(left[i], right[i]) << i;
  }
}

template <typename T>
static inline bool vector_equal(const std::vector<T>& left, const std::vector<T>& right) {
  if (left.size() != right.size()) {
    return false;
  }

  for (size_t i = 0; i < left.size(); ++i) {
    if (left[i] != right[i]) {
      std::cerr << "index " << i << " left was " << left[i] << " right was " << right[i]
                << std::endl;
      return false;
    }
  }

  return true;
}

template <typename T>
static std::vector<T> slice(const std::vector<T>& values, int start, int end) {
  if (end < start) {
    return std::vector<T>(0);
  }

  std::vector<T> out(end - start);
  for (int i = start; i < end; ++i) {
    out[i - start] = values[i];
  }
  return out;
}

void random_bytes(int n, uint32_t seed, std::vector<uint8_t>* out);
void random_bools(int n, double p, uint32_t seed, bool* out);

template <typename T>
inline void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) {
  std::default_random_engine gen(seed);
  std::uniform_int_distribution<T> d(min_value, max_value);
  for (int i = 0; i < n; ++i) {
    out[i] = d(gen);
  }
}

template <>
inline void random_numbers(int n, uint32_t seed, float min_value, float max_value,
                           float* out) {
  std::default_random_engine gen(seed);
  std::uniform_real_distribution<float> d(min_value, max_value);
  for (int i = 0; i < n; ++i) {
    out[i] = d(gen);
  }
}

template <>
inline void random_numbers(int n, uint32_t seed, double min_value, double max_value,
                           double* out) {
  std::default_random_engine gen(seed);
  std::uniform_real_distribution<double> d(min_value, max_value);
  for (int i = 0; i < n; ++i) {
    out[i] = d(gen);
  }
}

void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value,
                          Int96* out);

void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value,
                            ::arrow::util::Float16 max_value, uint16_t* out);

void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out);

void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size,
                       int max_size);

void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size);

void prefixed_random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out,
                                int min_size, int max_size, double prefixed_probability);

void prefixed_random_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out,
                                double prefixed_probability);

template <typename Type, typename Sequence>
std::shared_ptr<Buffer> EncodeValues(Encoding::type encoding, bool use_dictionary,
                                     const Sequence& values, int length,
                                     const ColumnDescriptor* descr) {
  auto encoder = MakeTypedEncoder<Type>(encoding, use_dictionary, descr);
  encoder->Put(values, length);
  return encoder->FlushValues();
}

template <typename T>
static void InitValues(int num_values, uint32_t seed, std::vector<T>& values,
                       std::vector<uint8_t>& buffer) {
  random_numbers(num_values, seed, std::numeric_limits<T>::min(),
                 std::numeric_limits<T>::max(), values.data());
}

template <typename T>
static void InitValues(int num_values, std::vector<T>& values,
                       std::vector<uint8_t>& buffer) {
  InitValues(num_values, 0, values, buffer);
}

template <typename T>
static void InitDictValues(int num_values, int num_dicts, std::vector<T>& values,
                           std::vector<uint8_t>& buffer) {
  int repeat_factor = num_values / num_dicts;
  InitValues<T>(num_dicts, values, buffer);
  // add some repeated values
  for (int j = 1; j < repeat_factor; ++j) {
    for (int i = 0; i < num_dicts; ++i) {
      std::memcpy(&values[num_dicts * j + i], &values[i], sizeof(T));
    }
  }
  // computed only dict_per_page * repeat_factor - 1 values < num_values
  // compute remaining
  for (int i = num_dicts * repeat_factor; i < num_values; ++i) {
    std::memcpy(&values[i], &values[i - num_dicts * repeat_factor], sizeof(T));
  }
}

template <>
inline void InitDictValues<bool>(int num_values, int num_dicts, std::vector<bool>& values,
                                 std::vector<uint8_t>& buffer) {
  // No op for bool
}

class MockPageReader : public PageReader {
 public:
  explicit MockPageReader(const std::vector<std::shared_ptr<Page>>& pages)
      : pages_(pages), page_index_(0) {}

  std::shared_ptr<Page> NextPage() override {
    if (page_index_ == static_cast<int>(pages_.size())) {
      // EOS to consumer
      return std::shared_ptr<Page>(nullptr);
    }
    return pages_[page_index_++];
  }

  // No-op
  void set_max_page_header_size(uint32_t size) override {}

 private:
  std::vector<std::shared_ptr<Page>> pages_;
  int page_index_;
};

// TODO(wesm): this is only used for testing for now. Refactor to form part of
// primary file write path
template <typename Type>
class DataPageBuilder {
 public:
  using c_type = typename Type::c_type;

  // This class writes data and metadata to the passed inputs
  explicit DataPageBuilder(ArrowOutputStream* sink)
      : sink_(sink),
        num_values_(0),
        encoding_(Encoding::PLAIN),
        definition_level_encoding_(Encoding::RLE),
        repetition_level_encoding_(Encoding::RLE),
        have_def_levels_(false),
        have_rep_levels_(false),
        have_values_(false) {}

  void AppendDefLevels(const std::vector<int16_t>& levels, int16_t max_level,
                       Encoding::type encoding = Encoding::RLE) {
    AppendLevels(levels, max_level, encoding);

    num_values_ = std::max(static_cast<int32_t>(levels.size()), num_values_);
    definition_level_encoding_ = encoding;
    have_def_levels_ = true;
  }

  void AppendRepLevels(const std::vector<int16_t>& levels, int16_t max_level,
                       Encoding::type encoding = Encoding::RLE) {
    AppendLevels(levels, max_level, encoding);

    num_values_ = std::max(static_cast<int32_t>(levels.size()), num_values_);
    repetition_level_encoding_ = encoding;
    have_rep_levels_ = true;
  }

  void AppendValues(const ColumnDescriptor* d, const std::vector<c_type>& values,
                    Encoding::type encoding = Encoding::PLAIN) {
    std::shared_ptr<Buffer> values_sink = EncodeValues<Type>(
        encoding, false, values.data(), static_cast<int>(values.size()), d);
    PARQUET_THROW_NOT_OK(sink_->Write(values_sink->data(), values_sink->size()));

    num_values_ = std::max(static_cast<int32_t>(values.size()), num_values_);
    encoding_ = encoding;
    have_values_ = true;
  }

  int32_t num_values() const { return num_values_; }

  Encoding::type encoding() const { return encoding_; }

  Encoding::type rep_level_encoding() const { return repetition_level_encoding_; }

  Encoding::type def_level_encoding() const { return definition_level_encoding_; }

 private:
  ArrowOutputStream* sink_;

  int32_t num_values_;
  Encoding::type encoding_;
  Encoding::type definition_level_encoding_;
  Encoding::type repetition_level_encoding_;

  bool have_def_levels_;
  bool have_rep_levels_;
  bool have_values_;

  // Used internally for both repetition and definition levels
  void AppendLevels(const std::vector<int16_t>& levels, int16_t max_level,
                    Encoding::type encoding) {
    if (encoding != Encoding::RLE) {
      ParquetException::NYI("only rle encoding currently implemented");
    }

    std::vector<uint8_t> encode_buffer(LevelEncoder::MaxBufferSize(
        Encoding::RLE, max_level, static_cast<int>(levels.size())));

    // We encode into separate memory from the output stream because the
    // RLE-encoded bytes have to be preceded in the stream by their absolute
    // size.
    LevelEncoder encoder;
    encoder.Init(encoding, max_level, static_cast<int>(levels.size()),
                 encode_buffer.data(), static_cast<int>(encode_buffer.size()));

    encoder.Encode(static_cast<int>(levels.size()), levels.data());

    int32_t rle_bytes = encoder.len();
    PARQUET_THROW_NOT_OK(
        sink_->Write(reinterpret_cast<const uint8_t*>(&rle_bytes), sizeof(int32_t)));
    PARQUET_THROW_NOT_OK(sink_->Write(encode_buffer.data(), rle_bytes));
  }
};

template <>
inline void DataPageBuilder<BooleanType>::AppendValues(const ColumnDescriptor* d,
                                                       const std::vector<bool>& values,
                                                       Encoding::type encoding) {
  if (encoding != Encoding::PLAIN) {
    ParquetException::NYI("only plain encoding currently implemented");
  }

  auto encoder = MakeTypedEncoder<BooleanType>(Encoding::PLAIN, false, d);
  dynamic_cast<BooleanEncoder*>(encoder.get())
      ->Put(values, static_cast<int>(values.size()));
  std::shared_ptr<Buffer> buffer = encoder->FlushValues();
  PARQUET_THROW_NOT_OK(sink_->Write(buffer->data(), buffer->size()));

  num_values_ = std::max(static_cast<int32_t>(values.size()), num_values_);
  encoding_ = encoding;
  have_values_ = true;
}

template <typename Type>
static std::shared_ptr<DataPageV1> MakeDataPage(
    const ColumnDescriptor* d, const std::vector<typename Type::c_type>& values,
    int num_vals, Encoding::type encoding, const uint8_t* indices, int indices_size,
    const std::vector<int16_t>& def_levels, int16_t max_def_level,
    const std::vector<int16_t>& rep_levels, int16_t max_rep_level) {
  int num_values = 0;

  auto page_stream = CreateOutputStream();
  test::DataPageBuilder<Type> page_builder(page_stream.get());

  if (!rep_levels.empty()) {
    page_builder.AppendRepLevels(rep_levels, max_rep_level);
  }
  if (!def_levels.empty()) {
    page_builder.AppendDefLevels(def_levels, max_def_level);
  }

  if (encoding == Encoding::PLAIN) {
    page_builder.AppendValues(d, values, encoding);
    num_values = std::max(page_builder.num_values(), num_vals);
  } else {  // DICTIONARY PAGES
    PARQUET_THROW_NOT_OK(page_stream->Write(indices, indices_size));
    num_values = std::max(page_builder.num_values(), num_vals);
  }

  PARQUET_ASSIGN_OR_THROW(auto buffer, page_stream->Finish());

  return std::make_shared<DataPageV1>(buffer, num_values, encoding,
                                      page_builder.def_level_encoding(),
                                      page_builder.rep_level_encoding(), buffer->size());
}

template <typename TYPE>
class DictionaryPageBuilder {
 public:
  typedef typename TYPE::c_type TC;
  static constexpr int TN = TYPE::type_num;
  using SpecializedEncoder = typename EncodingTraits<TYPE>::Encoder;

  // This class writes data and metadata to the passed inputs
  explicit DictionaryPageBuilder(const ColumnDescriptor* d)
      : num_dict_values_(0), have_values_(false) {
    auto encoder = MakeTypedEncoder<TYPE>(Encoding::PLAIN, true, d);
    dict_traits_ = dynamic_cast<DictEncoder<TYPE>*>(encoder.get());
    encoder_.reset(dynamic_cast<SpecializedEncoder*>(encoder.release()));
  }

  ~DictionaryPageBuilder() {}

  std::shared_ptr<Buffer> AppendValues(const std::vector<TC>& values) {
    int num_values = static_cast<int>(values.size());
    // Dictionary encoding
    encoder_->Put(values.data(), num_values);
    num_dict_values_ = dict_traits_->num_entries();
    have_values_ = true;
    return encoder_->FlushValues();
  }

  std::shared_ptr<Buffer> WriteDict() {
    std::shared_ptr<Buffer> dict_buffer =
        AllocateBuffer(::arrow::default_memory_pool(), dict_traits_->dict_encoded_size());
    dict_traits_->WriteDict(dict_buffer->mutable_data());
    return dict_buffer;
  }

  int32_t num_values() const { return num_dict_values_; }

 private:
  DictEncoder<TYPE>* dict_traits_;
  std::unique_ptr<SpecializedEncoder> encoder_;
  int32_t num_dict_values_;
  bool have_values_;
};

template <>
inline DictionaryPageBuilder<BooleanType>::DictionaryPageBuilder(
    const ColumnDescriptor* d) {
  ParquetException::NYI("only plain encoding currently implemented for boolean");
}

template <>
inline std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::WriteDict() {
  ParquetException::NYI("only plain encoding currently implemented for boolean");
  return nullptr;
}

template <>
inline std::shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::AppendValues(
    const std::vector<TC>& values) {
  ParquetException::NYI("only plain encoding currently implemented for boolean");
  return nullptr;
}

template <typename Type>
inline static std::shared_ptr<DictionaryPage> MakeDictPage(
    const ColumnDescriptor* d, const std::vector<typename Type::c_type>& values,
    const std::vector<int>& values_per_page, Encoding::type encoding,
    std::vector<std::shared_ptr<Buffer>>& rle_indices) {
  test::DictionaryPageBuilder<Type> page_builder(d);
  int num_pages = static_cast<int>(values_per_page.size());
  int value_start = 0;

  for (int i = 0; i < num_pages; i++) {
    rle_indices.push_back(page_builder.AppendValues(
        slice(values, value_start, value_start + values_per_page[i])));
    value_start += values_per_page[i];
  }

  auto buffer = page_builder.WriteDict();

  return std::make_shared<DictionaryPage>(buffer, page_builder.num_values(),
                                          Encoding::PLAIN);
}

// Given def/rep levels and values create multiple dict pages
template <typename Type>
inline static void PaginateDict(const ColumnDescriptor* d,
                                const std::vector<typename Type::c_type>& values,
                                const std::vector<int16_t>& def_levels,
                                int16_t max_def_level,
                                const std::vector<int16_t>& rep_levels,
                                int16_t max_rep_level, int num_levels_per_page,
                                const std::vector<int>& values_per_page,
                                std::vector<std::shared_ptr<Page>>& pages,
                                Encoding::type encoding = Encoding::RLE_DICTIONARY) {
  int num_pages = static_cast<int>(values_per_page.size());
  std::vector<std::shared_ptr<Buffer>> rle_indices;
  std::shared_ptr<DictionaryPage> dict_page =
      MakeDictPage<Type>(d, values, values_per_page, encoding, rle_indices);
  pages.push_back(dict_page);
  int def_level_start = 0;
  int def_level_end = 0;
  int rep_level_start = 0;
  int rep_level_end = 0;
  for (int i = 0; i < num_pages; i++) {
    if (max_def_level > 0) {
      def_level_start = i * num_levels_per_page;
      def_level_end = (i + 1) * num_levels_per_page;
    }
    if (max_rep_level > 0) {
      rep_level_start = i * num_levels_per_page;
      rep_level_end = (i + 1) * num_levels_per_page;
    }
    std::shared_ptr<DataPageV1> data_page = MakeDataPage<Int32Type>(
        d, {}, values_per_page[i], encoding, rle_indices[i]->data(),
        static_cast<int>(rle_indices[i]->size()),
        slice(def_levels, def_level_start, def_level_end), max_def_level,
        slice(rep_levels, rep_level_start, rep_level_end), max_rep_level);
    pages.push_back(data_page);
  }
}

// Given def/rep levels and values create multiple plain pages
template <typename Type>
static inline void PaginatePlain(const ColumnDescriptor* d,
                                 const std::vector<typename Type::c_type>& values,
                                 const std::vector<int16_t>& def_levels,
                                 int16_t max_def_level,
                                 const std::vector<int16_t>& rep_levels,
                                 int16_t max_rep_level, int num_levels_per_page,
                                 const std::vector<int>& values_per_page,
                                 std::vector<std::shared_ptr<Page>>& pages,
                                 Encoding::type encoding = Encoding::PLAIN) {
  int num_pages = static_cast<int>(values_per_page.size());
  int def_level_start = 0;
  int def_level_end = 0;
  int rep_level_start = 0;
  int rep_level_end = 0;
  int value_start = 0;
  for (int i = 0; i < num_pages; i++) {
    if (max_def_level > 0) {
      def_level_start = i * num_levels_per_page;
      def_level_end = (i + 1) * num_levels_per_page;
    }
    if (max_rep_level > 0) {
      rep_level_start = i * num_levels_per_page;
      rep_level_end = (i + 1) * num_levels_per_page;
    }
    std::shared_ptr<DataPage> page = MakeDataPage<Type>(
        d, slice(values, value_start, value_start + values_per_page[i]),
        values_per_page[i], encoding, nullptr, 0,
        slice(def_levels, def_level_start, def_level_end), max_def_level,
        slice(rep_levels, rep_level_start, rep_level_end), max_rep_level);
    pages.push_back(page);
    value_start += values_per_page[i];
  }
}

// Generates pages from randomly generated data
template <typename Type>
static inline int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page,
                            std::vector<int16_t>& def_levels,
                            std::vector<int16_t>& rep_levels,
                            std::vector<typename Type::c_type>& values,
                            std::vector<uint8_t>& buffer,
                            std::vector<std::shared_ptr<Page>>& pages,
                            Encoding::type encoding = Encoding::PLAIN,
                            uint32_t seed = 0) {
  int num_levels = levels_per_page * num_pages;
  int num_values = 0;
  int16_t zero = 0;
  int16_t max_def_level = d->max_definition_level();
  int16_t max_rep_level = d->max_repetition_level();
  std::vector<int> values_per_page(num_pages, levels_per_page);
  // Create definition levels
  if (max_def_level > 0 && num_levels != 0) {
    def_levels.resize(num_levels);
    random_numbers(num_levels, seed, zero, max_def_level, def_levels.data());
    for (int p = 0; p < num_pages; p++) {
      int num_values_per_page = 0;
      for (int i = 0; i < levels_per_page; i++) {
        if (def_levels[i + p * levels_per_page] == max_def_level) {
          num_values_per_page++;
          num_values++;
        }
      }
      values_per_page[p] = num_values_per_page;
    }
  } else {
    num_values = num_levels;
  }
  // Create repetition levels
  if (max_rep_level > 0 && num_levels != 0) {
    rep_levels.resize(num_levels);
    // Using a different seed so that def_levels and rep_levels are different.
    random_numbers(num_levels, seed + 789, zero, max_rep_level, rep_levels.data());
    // The generated levels are random. Force the very first page to start with a new
    // record.
    rep_levels[0] = 0;
    // For a null value, rep_levels and def_levels are both 0.
    // If we have a repeated value right after this, it needs to start with
    // rep_level = 0 to indicate a new record.
    for (int i = 0; i < num_levels - 1; ++i) {
      if (rep_levels[i] == 0 && def_levels[i] == 0) {
        rep_levels[i + 1] = 0;
      }
    }
  }
  // Create values
  values.resize(num_values);
  if (encoding == Encoding::PLAIN) {
    InitValues<typename Type::c_type>(num_values, values, buffer);
    PaginatePlain<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
                        levels_per_page, values_per_page, pages);
  } else if (encoding == Encoding::RLE_DICTIONARY ||
             encoding == Encoding::PLAIN_DICTIONARY) {
    // Calls InitValues and repeats the data
    InitDictValues<typename Type::c_type>(num_values, levels_per_page, values, buffer);
    PaginateDict<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
                       levels_per_page, values_per_page, pages);
  }

  return num_values;
}

// ----------------------------------------------------------------------
// Test data generation

template <>
void inline InitValues<bool>(int num_values, uint32_t seed, std::vector<bool>& values,
                             std::vector<uint8_t>& buffer) {
  values = {};
  if (seed == 0) {
    seed = static_cast<uint32_t>(::arrow::random_seed());
  }
  ::arrow::random_is_valid(num_values, 0.5, &values, static_cast<int>(seed));
}

template <>
inline void InitValues<ByteArray>(int num_values, uint32_t seed,
                                  std::vector<ByteArray>& values,
                                  std::vector<uint8_t>& buffer) {
  int max_byte_array_len = 12;
  int num_bytes = static_cast<int>(max_byte_array_len + sizeof(uint32_t));
  size_t nbytes = num_values * num_bytes;
  buffer.resize(nbytes);
  random_byte_array(num_values, seed, buffer.data(), values.data(), max_byte_array_len);
}

inline void InitWideByteArrayValues(int num_values, std::vector<ByteArray>& values,
                                    std::vector<uint8_t>& buffer, int min_len,
                                    int max_len) {
  int num_bytes = static_cast<int>(max_len + sizeof(uint32_t));
  size_t nbytes = num_values * num_bytes;
  buffer.resize(nbytes);
  random_byte_array(num_values, 0, buffer.data(), values.data(), min_len, max_len);
}

template <>
inline void InitValues<FLBA>(int num_values, uint32_t seed, std::vector<FLBA>& values,
                             std::vector<uint8_t>& buffer) {
  size_t nbytes = num_values * FLBA_LENGTH;
  buffer.resize(nbytes);
  random_fixed_byte_array(num_values, seed, buffer.data(), FLBA_LENGTH, values.data());
}

template <>
inline void InitValues<Int96>(int num_values, uint32_t seed, std::vector<Int96>& values,
                              std::vector<uint8_t>& buffer) {
  random_Int96_numbers(num_values, seed, std::numeric_limits<int32_t>::min(),
                       std::numeric_limits<int32_t>::max(), values.data());
}

inline std::string TestColumnName(int i) {
  std::stringstream col_name;
  col_name << "column_" << i;
  return col_name.str();
}

// This class lives here because of its dependency on the InitValues specializations.
template <typename TestType>
class PrimitiveTypedTest : public ::testing::Test {
 public:
  using c_type = typename TestType::c_type;

  void SetUpSchema(Repetition::type repetition, int num_columns = 1) {
    std::vector<schema::NodePtr> fields;

    for (int i = 0; i < num_columns; ++i) {
      std::string name = TestColumnName(i);
      fields.push_back(schema::PrimitiveNode::Make(name, repetition, TestType::type_num,
                                                   ConvertedType::NONE, FLBA_LENGTH));
    }
    node_ = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields);
    schema_.Init(node_);
  }

  void GenerateData(int64_t num_values, uint32_t seed = 0);
  void SetupValuesOut(int64_t num_values);
  void SyncValuesOut();

 protected:
  schema::NodePtr node_;
  SchemaDescriptor schema_;

  // Input buffers
  std::vector<c_type> values_;

  std::vector<int16_t> def_levels_;

  std::vector<uint8_t> buffer_;
  // Pointer to the values, needed as we cannot use std::vector<bool>::data()
  c_type* values_ptr_;
  std::vector<uint8_t> bool_buffer_;

  // Output buffers
  std::vector<c_type> values_out_;
  std::vector<uint8_t> bool_buffer_out_;
  c_type* values_out_ptr_;
};

template <typename TestType>
inline void PrimitiveTypedTest<TestType>::SyncValuesOut() {}

template <>
inline void PrimitiveTypedTest<BooleanType>::SyncValuesOut() {
  std::vector<uint8_t>::const_iterator source_iterator = bool_buffer_out_.begin();
  std::vector<c_type>::iterator destination_iterator = values_out_.begin();
  while (source_iterator != bool_buffer_out_.end()) {
    *destination_iterator++ = *source_iterator++ != 0;
  }
}

template <typename TestType>
inline void PrimitiveTypedTest<TestType>::SetupValuesOut(int64_t num_values) {
  values_out_.clear();
  values_out_.resize(num_values);
  values_out_ptr_ = values_out_.data();
}

template <>
inline void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) {
  values_out_.clear();
  values_out_.resize(num_values);

  bool_buffer_out_.clear();
  bool_buffer_out_.resize(num_values);
  // Write once to all values so we can copy it without getting Valgrind errors
  // about uninitialised values.
  std::fill(bool_buffer_out_.begin(), bool_buffer_out_.end(), true);
  values_out_ptr_ = reinterpret_cast<bool*>(bool_buffer_out_.data());
}

template <typename TestType>
inline void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values,
                                                       uint32_t seed) {
  def_levels_.resize(num_values);
  values_.resize(num_values);

  InitValues<c_type>(static_cast<int>(num_values), seed, values_, buffer_);
  values_ptr_ = values_.data();

  std::fill(def_levels_.begin(), def_levels_.end(), 1);
}

template <>
inline void PrimitiveTypedTest<BooleanType>::GenerateData(int64_t num_values,
                                                          uint32_t seed) {
  def_levels_.resize(num_values);
  values_.resize(num_values);

  InitValues<c_type>(static_cast<int>(num_values), seed, values_, buffer_);
  bool_buffer_.resize(num_values);
  std::copy(values_.begin(), values_.end(), bool_buffer_.begin());
  values_ptr_ = reinterpret_cast<bool*>(bool_buffer_.data());

  std::fill(def_levels_.begin(), def_levels_.end(), 1);
}

// ----------------------------------------------------------------------
// test data generation

template <typename T>
inline void GenerateData(int num_values, T* out, std::vector<uint8_t>* heap) {
  // seed the prng so failure is deterministic
  random_numbers(num_values, 0, std::numeric_limits<T>::min(),
                 std::numeric_limits<T>::max(), out);
}

template <typename T>
inline void GenerateBoundData(int num_values, T* out, T min, T max,
                              std::vector<uint8_t>* heap) {
  // seed the prng so failure is deterministic
  random_numbers(num_values, 0, min, max, out);
}

template <>
inline void GenerateData<bool>(int num_values, bool* out, std::vector<uint8_t>* heap) {
  // seed the prng so failure is deterministic
  random_bools(num_values, 0.5, 0, out);
}

template <>
inline void GenerateData<Int96>(int num_values, Int96* out, std::vector<uint8_t>* heap) {
  // seed the prng so failure is deterministic
  random_Int96_numbers(num_values, 0, std::numeric_limits<int32_t>::min(),
                       std::numeric_limits<int32_t>::max(), out);
}

template <>
inline void GenerateData<ByteArray>(int num_values, ByteArray* out,
                                    std::vector<uint8_t>* heap) {
  int max_byte_array_len = 12;
  heap->resize(num_values * max_byte_array_len);
  // seed the prng so failure is deterministic
  random_byte_array(num_values, 0, heap->data(), out, 2, max_byte_array_len);
}

// Generate ByteArray or FLBA data where there is a given probability
// for each value to share a common prefix with its predecessor.
// This is useful to exercise prefix-based encodings such as DELTA_BYTE_ARRAY.
template <typename T>
inline void GeneratePrefixedData(int num_values, T* out, std::vector<uint8_t>* heap,
                                 double prefixed_probability);

template <>
inline void GeneratePrefixedData(int num_values, ByteArray* out,
                                 std::vector<uint8_t>* heap,
                                 double prefixed_probability) {
  int max_byte_array_len = 12;
  heap->resize(num_values * max_byte_array_len);
  // seed the prng so failure is deterministic
  prefixed_random_byte_array(num_values, /*seed=*/0, heap->data(), out, /*min_size=*/2,
                             /*max_size=*/max_byte_array_len, prefixed_probability);
}

static constexpr int kGenerateDataFLBALength = 8;

template <>
inline void GeneratePrefixedData<FLBA>(int num_values, FLBA* out,
                                       std::vector<uint8_t>* heap,
                                       double prefixed_probability) {
  heap->resize(num_values * kGenerateDataFLBALength);
  // seed the prng so failure is deterministic
  prefixed_random_byte_array(num_values, /*seed=*/0, heap->data(),
                             kGenerateDataFLBALength, out, prefixed_probability);
}

template <>
inline void GenerateData<FLBA>(int num_values, FLBA* out, std::vector<uint8_t>* heap) {
  heap->resize(num_values * kGenerateDataFLBALength);
  // seed the prng so failure is deterministic
  random_fixed_byte_array(num_values, 0, heap->data(), kGenerateDataFLBALength, out);
}

}  // namespace test
}  // namespace parquet