|
|
# Licensed to the Apache Software Foundation (ASF) under one |
|
|
# or more contributor license agreements. See the NOTICE file |
|
|
# distributed with this work for additional information |
|
|
# regarding copyright ownership. The ASF licenses this file |
|
|
# to you under the Apache License, Version 2.0 (the |
|
|
# "License"); you may not use this file except in compliance |
|
|
# with the License. You may obtain a copy of the License at |
|
|
# |
|
|
# http: |
|
|
# |
|
|
# Unless required by applicable law or agreed to in writing, |
|
|
# software distributed under the License is distributed on an |
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
|
|
# KIND, either express or implied. See the License for the |
|
|
# specific language governing permissions and limitations |
|
|
# under the License. |
|
|
|
|
|
# cython: language_level = 3 |
|
|
|
|
|
"""Dataset is currently unstable. APIs subject to change without notice.""" |
|
|
|
|
|
from pyarrow.includes.common cimport * |
|
|
from pyarrow.includes.libarrow_dataset cimport * |
|
|
from pyarrow.lib cimport * |
|
|
from pyarrow._fs cimport FileSystem, FileInfo |
|
|
|
|
|
|
|
|
cdef CFileSource _make_file_source(object file, FileSystem filesystem=*, object file_size=*) |
|
|
|
|
|
cdef class DatasetFactory(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
SharedPtrNoGIL[CDatasetFactory] wrapped |
|
|
CDatasetFactory* factory |
|
|
|
|
|
cdef init(self, const shared_ptr[CDatasetFactory]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CDatasetFactory]& sp) |
|
|
|
|
|
cdef inline shared_ptr[CDatasetFactory] unwrap(self) nogil |
|
|
|
|
|
|
|
|
cdef class Dataset(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
SharedPtrNoGIL[CDataset] wrapped |
|
|
CDataset* dataset |
|
|
public dict _scan_options |
|
|
|
|
|
cdef void init(self, const shared_ptr[CDataset]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CDataset]& sp) |
|
|
|
|
|
cdef shared_ptr[CDataset] unwrap(self) nogil |
|
|
|
|
|
|
|
|
cdef class Scanner(_Weakrefable): |
|
|
cdef: |
|
|
SharedPtrNoGIL[CScanner] wrapped |
|
|
CScanner* scanner |
|
|
|
|
|
cdef void init(self, const shared_ptr[CScanner]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CScanner]& sp) |
|
|
|
|
|
cdef shared_ptr[CScanner] unwrap(self) |
|
|
|
|
|
@staticmethod |
|
|
cdef shared_ptr[CScanOptions] _make_scan_options(Dataset dataset, dict py_scanoptions) except * |
|
|
|
|
|
|
|
|
cdef class FragmentScanOptions(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
shared_ptr[CFragmentScanOptions] wrapped |
|
|
|
|
|
cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CFragmentScanOptions]& sp) |
|
|
|
|
|
|
|
|
cdef class FileFormat(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
shared_ptr[CFileFormat] wrapped |
|
|
CFileFormat* format |
|
|
|
|
|
cdef void init(self, const shared_ptr[CFileFormat]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CFileFormat]& sp) |
|
|
|
|
|
cdef inline shared_ptr[CFileFormat] unwrap(self) |
|
|
|
|
|
cdef _set_default_fragment_scan_options(self, FragmentScanOptions options) |
|
|
|
|
|
# Return a WrittenFile after a file was written. |
|
|
# May be overridden by subclasses, e.g. to add metadata. |
|
|
cdef WrittenFile _finish_write(self, path, base_dir, |
|
|
CFileWriter* file_writer) |
|
|
|
|
|
|
|
|
cdef class FileWriteOptions(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
shared_ptr[CFileWriteOptions] wrapped |
|
|
CFileWriteOptions* c_options |
|
|
|
|
|
cdef void init(self, const shared_ptr[CFileWriteOptions]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CFileWriteOptions]& sp) |
|
|
|
|
|
cdef inline shared_ptr[CFileWriteOptions] unwrap(self) |
|
|
|
|
|
|
|
|
cdef class Fragment(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
SharedPtrNoGIL[CFragment] wrapped |
|
|
CFragment* fragment |
|
|
|
|
|
cdef void init(self, const shared_ptr[CFragment]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CFragment]& sp) |
|
|
|
|
|
cdef inline shared_ptr[CFragment] unwrap(self) |
|
|
|
|
|
|
|
|
cdef class FileFragment(Fragment): |
|
|
|
|
|
cdef: |
|
|
CFileFragment* file_fragment |
|
|
|
|
|
cdef void init(self, const shared_ptr[CFragment]& sp) |
|
|
|
|
|
|
|
|
cdef class Partitioning(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
shared_ptr[CPartitioning] wrapped |
|
|
CPartitioning* partitioning |
|
|
|
|
|
cdef init(self, const shared_ptr[CPartitioning]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CPartitioning]& sp) |
|
|
|
|
|
cdef inline shared_ptr[CPartitioning] unwrap(self) |
|
|
|
|
|
|
|
|
cdef class PartitioningFactory(_Weakrefable): |
|
|
|
|
|
cdef: |
|
|
shared_ptr[CPartitioningFactory] wrapped |
|
|
CPartitioningFactory* factory |
|
|
object constructor |
|
|
object options |
|
|
|
|
|
cdef init(self, const shared_ptr[CPartitioningFactory]& sp) |
|
|
|
|
|
@staticmethod |
|
|
cdef wrap(const shared_ptr[CPartitioningFactory]& sp, |
|
|
object constructor, object options) |
|
|
|
|
|
cdef inline shared_ptr[CPartitioningFactory] unwrap(self) |
|
|
|
|
|
|
|
|
cdef class WrittenFile(_Weakrefable): |
|
|
|
|
|
# The full path to the created file |
|
|
cdef public str path |
|
|
# Optional Parquet metadata |
|
|
# This metadata will have the file path attribute set to the path of |
|
|
# the written file. |
|
|
cdef public object metadata |
|
|
# The size of the file in bytes |
|
|
cdef public int64_t size |
|
|
|
