# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # distutils: language = c++ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_python cimport CTimePoint cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: ctypedef enum CFileType "arrow::fs::FileType": CFileType_NotFound "arrow::fs::FileType::NotFound" CFileType_Unknown "arrow::fs::FileType::Unknown" CFileType_File "arrow::fs::FileType::File" CFileType_Directory "arrow::fs::FileType::Directory" cdef cppclass CFileInfo "arrow::fs::FileInfo": CFileInfo() CFileInfo(CFileInfo) CFileInfo& operator=(CFileInfo) CFileInfo(const CFileInfo&) CFileInfo& operator=(const CFileInfo&) CFileType type() void set_type(CFileType type) c_string path() void set_path(const c_string& path) c_string base_name() int64_t size() void set_size(int64_t size) c_string extension() CTimePoint mtime() void set_mtime(CTimePoint mtime) cdef cppclass CFileSelector "arrow::fs::FileSelector": CFileSelector() c_string base_dir c_bool allow_not_found c_bool recursive cdef cppclass CFileLocator "arrow::fs::FileLocator": shared_ptr[CFileSystem] filesystem c_string path cdef cppclass CFileSystem "arrow::fs::FileSystem": shared_ptr[CFileSystem] shared_from_this() c_string type_name() const CResult[c_string] NormalizePath(c_string path) CResult[c_string] MakeUri(c_string path) CResult[CFileInfo] GetFileInfo(const c_string& path) CResult[vector[CFileInfo]] GetFileInfo( const vector[c_string]& paths) CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select) CStatus CreateDir(const c_string& path, c_bool recursive) CStatus DeleteDir(const c_string& path) CStatus DeleteDirContents(const c_string& path, c_bool missing_dir_ok) CStatus DeleteRootDirContents() CStatus DeleteFile(const c_string& path) CStatus DeleteFiles(const vector[c_string]& paths) CStatus Move(const c_string& src, const c_string& dest) CStatus CopyFile(const c_string& src, const c_string& dest) CResult[shared_ptr[CInputStream]] OpenInputStream( const c_string& path) CResult[shared_ptr[CRandomAccessFile]] OpenInputFile( const c_string& path) CResult[shared_ptr[COutputStream]] OpenOutputStream( const c_string& path, const shared_ptr[const CKeyValueMetadata]&) CResult[shared_ptr[COutputStream]] OpenAppendStream( const c_string& path, const shared_ptr[const CKeyValueMetadata]&) c_bool Equals(const CFileSystem& other) c_bool Equals(shared_ptr[CFileSystem] other) CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \ "arrow::fs::FileSystemFromUri"(const c_string& uri) CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \ "arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path) CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \ "arrow::fs::FileSystemFromUriOrPath"(const c_string& uri, c_string* out_path) cdef cppclass CFileSystemGlobalOptions \ "arrow::fs::FileSystemGlobalOptions": c_string tls_ca_file_path c_string tls_ca_dir_path CStatus CFileSystemsInitialize "arrow::fs::Initialize" \ (const CFileSystemGlobalOptions& options) cdef cppclass CSubTreeFileSystem \ "arrow::fs::SubTreeFileSystem"(CFileSystem): CSubTreeFileSystem(const c_string& base_path, shared_ptr[CFileSystem] base_fs) c_string base_path() shared_ptr[CFileSystem] base_fs() ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": CS3LogLevel log_level int num_event_loop_threads cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions": c_string scheme c_string host int port c_string username c_string password c_bool Equals(const CS3ProxyOptions& other) @staticmethod CResult[CS3ProxyOptions] FromUriString "FromUri"( const c_string& uri_string) ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind": CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous" CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default" CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit" CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role" CS3CredentialsKind_WebIdentity \ "arrow::fs::S3CredentialsKind::WebIdentity" cdef cppclass CS3RetryStrategy "arrow::fs::S3RetryStrategy": @staticmethod shared_ptr[CS3RetryStrategy] GetAwsDefaultRetryStrategy(int64_t max_attempts) @staticmethod shared_ptr[CS3RetryStrategy] GetAwsStandardRetryStrategy(int64_t max_attempts) cdef cppclass CS3Options "arrow::fs::S3Options": c_string region double connect_timeout double request_timeout c_string endpoint_override c_string scheme c_bool background_writes c_bool allow_bucket_creation c_bool allow_bucket_deletion c_bool check_directory_existence_before_creation c_bool force_virtual_addressing shared_ptr[const CKeyValueMetadata] default_metadata c_string role_arn c_string session_name c_string external_id int load_frequency CS3ProxyOptions proxy_options CS3CredentialsKind credentials_kind shared_ptr[CS3RetryStrategy] retry_strategy void ConfigureDefaultCredentials() void ConfigureAccessKey(const c_string& access_key, const c_string& secret_key, const c_string& session_token) c_string GetAccessKey() c_string GetSecretKey() c_string GetSessionToken() c_bool Equals(const CS3Options& other) @staticmethod CS3Options Defaults() @staticmethod CS3Options Anonymous() @staticmethod CS3Options FromAccessKey(const c_string& access_key, const c_string& secret_key, const c_string& session_token) @staticmethod CS3Options FromAssumeRole(const c_string& role_arn, const c_string& session_name, const c_string& external_id, const int load_frequency) cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): @staticmethod CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options) CS3Options options() c_string region() cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( const CS3GlobalOptions& options) cdef CStatus CEnsureS3Initialized "arrow::fs::EnsureS3Initialized"() cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() cdef CStatus CEnsureS3Finalized "arrow::fs::EnsureS3Finalized"() cdef CResult[c_string] ResolveS3BucketRegion(const c_string& bucket) cdef cppclass CGcsCredentials "arrow::fs::GcsCredentials": c_bool anonymous() CTimePoint expiration() c_string access_token() c_string target_service_account() cdef cppclass CGcsOptions "arrow::fs::GcsOptions": CGcsCredentials credentials c_string endpoint_override c_string scheme c_string default_bucket_location optional[c_string] project_id optional[double] retry_limit_seconds shared_ptr[const CKeyValueMetadata] default_metadata c_bool Equals(const CS3Options& other) @staticmethod CGcsOptions Defaults() @staticmethod CGcsOptions Anonymous() @staticmethod CGcsOptions FromAccessToken(const c_string& access_token, CTimePoint expiration) @staticmethod CGcsOptions FromImpersonatedServiceAccount(const CGcsCredentials& base_credentials, c_string& target_service_account) cdef cppclass CGcsFileSystem "arrow::fs::GcsFileSystem": @staticmethod CResult[shared_ptr[CGcsFileSystem]] Make(const CGcsOptions& options) CGcsOptions options() cdef cppclass CAzureOptions "arrow::fs::AzureOptions": c_string account_name c_string blob_storage_authority c_string dfs_storage_authority c_string blob_storage_scheme c_string dfs_storage_scheme c_bool Equals(const CAzureOptions& other) CStatus ConfigureDefaultCredential() CStatus ConfigureAccountKeyCredential(c_string account_key) CStatus ConfigureSASCredential(c_string sas_token) cdef cppclass CAzureFileSystem "arrow::fs::AzureFileSystem": @staticmethod CResult[shared_ptr[CAzureFileSystem]] Make(const CAzureOptions& options) CAzureOptions options() cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions": HdfsConnectionConfig connection_config int32_t buffer_size int16_t replication int64_t default_block_size @staticmethod CResult[CHdfsOptions] FromUriString "FromUri"( const c_string& uri_string) void ConfigureEndPoint(c_string host, int port) void ConfigureDriver(c_bool use_hdfs3) void ConfigureReplication(int16_t replication) void ConfigureUser(c_string user_name) void ConfigureBufferSize(int32_t buffer_size) void ConfigureBlockSize(int64_t default_block_size) void ConfigureKerberosTicketCachePath(c_string path) void ConfigureExtraConf(c_string key, c_string value) cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem): @staticmethod CResult[shared_ptr[CHadoopFileSystem]] Make( const CHdfsOptions& options) CHdfsOptions options() cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"( CFileSystem): CMockFileSystem(CTimePoint current_time) CStatus CCopyFiles "arrow::fs::CopyFiles"( const vector[CFileLocator]& sources, const vector[CFileLocator]& destinations, const CIOContext& io_context, int64_t chunk_size, c_bool use_threads) CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"( const shared_ptr[CFileSystem]& source_fs, const CFileSelector& source_sel, const shared_ptr[CFileSystem]& destination_fs, const c_string& destination_base_dir, const CIOContext& io_context, int64_t chunk_size, c_bool use_threads) # Callbacks for implementing Python filesystems # Use typedef to emulate syntax for std::function ctypedef void CallbackGetTypeName(object, c_string*) ctypedef c_bool CallbackEquals(object, const CFileSystem&) ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*) ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&, vector[CFileInfo]*) ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&, vector[CFileInfo]*) ctypedef void CallbackCreateDir(object, const c_string&, c_bool) ctypedef void CallbackDeleteDir(object, const c_string&) ctypedef void CallbackDeleteDirContents(object, const c_string&, c_bool) ctypedef void CallbackDeleteRootDirContents(object) ctypedef void CallbackDeleteFile(object, const c_string&) ctypedef void CallbackMove(object, const c_string&, const c_string&) ctypedef void CallbackCopyFile(object, const c_string&, const c_string&) ctypedef void CallbackOpenInputStream(object, const c_string&, shared_ptr[CInputStream]*) ctypedef void CallbackOpenInputFile(object, const c_string&, shared_ptr[CRandomAccessFile]*) ctypedef void CallbackOpenOutputStream( object, const c_string&, const shared_ptr[const CKeyValueMetadata]&, shared_ptr[COutputStream]*) ctypedef void CallbackNormalizePath(object, const c_string&, c_string*) cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil: cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable": PyFileSystemVtable() function[CallbackGetTypeName] get_type_name function[CallbackEquals] equals function[CallbackGetFileInfo] get_file_info function[CallbackGetFileInfoVector] get_file_info_vector function[CallbackGetFileInfoSelector] get_file_info_selector function[CallbackCreateDir] create_dir function[CallbackDeleteDir] delete_dir function[CallbackDeleteDirContents] delete_dir_contents function[CallbackDeleteRootDirContents] delete_root_dir_contents function[CallbackDeleteFile] delete_file function[CallbackMove] move function[CallbackCopyFile] copy_file function[CallbackOpenInputStream] open_input_stream function[CallbackOpenInputFile] open_input_file function[CallbackOpenOutputStream] open_output_stream function[CallbackOpenOutputStream] open_append_stream function[CallbackNormalizePath] normalize_path cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem": @staticmethod shared_ptr[CPyFileSystem] Make(object handler, CPyFileSystemVtable vtable) PyObject* handler()