polars_io/parquet/read/
options.rs

1use std::fmt::Debug;
2use std::hash::Hash;
3use std::sync::Arc;
4
5use polars_core::schema::SchemaRef;
6use polars_parquet::parquet::encryption::decrypt::FileDecryptionProperties;
7#[cfg(feature = "serde")]
8use serde::{Deserialize, Serialize};
9
10#[derive(Clone, Debug, PartialEq, Eq, Hash)]
11#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
12#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
13pub struct ParquetOptions {
14    pub schema: Option<SchemaRef>,
15    pub parallel: ParallelStrategy,
16    pub low_memory: bool,
17    pub use_statistics: bool,
18    #[cfg_attr(feature = "serde", serde(default))]
19    pub decryption_properties: Option<PlFileDecryptionProperties>,
20}
21
22impl Default for ParquetOptions {
23    fn default() -> Self {
24        Self {
25            schema: None,
26            parallel: ParallelStrategy::default(),
27            low_memory: false,
28            use_statistics: true,
29            decryption_properties: None,
30        }
31    }
32}
33
34/// Properties for reading files encrypted with Parquet modular encryption.
35///
36/// These hold secret keys, so can't be serialized, and are compared and hashed
37/// by pointer.
38#[derive(Clone)]
39pub struct PlFileDecryptionProperties(pub Arc<FileDecryptionProperties>);
40
41impl Debug for PlFileDecryptionProperties {
42    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
43        // FileDecryptionProperties doesn't output keys.
44        self.0.fmt(f)
45    }
46}
47
48impl Eq for PlFileDecryptionProperties {}
49
50impl PartialEq for PlFileDecryptionProperties {
51    fn eq(&self, other: &Self) -> bool {
52        Arc::ptr_eq(&self.0, &other.0)
53    }
54}
55
56impl Hash for PlFileDecryptionProperties {
57    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
58        state.write_usize(Arc::as_ptr(&self.0) as usize)
59    }
60}
61
62#[cfg(feature = "serde")]
63impl<'de> Deserialize<'de> for PlFileDecryptionProperties {
64    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
65    where
66        D: serde::Deserializer<'de>,
67    {
68        use serde::de::Error;
69        Err(D::Error::custom(
70            "cannot deserialize parquet decryption properties",
71        ))
72    }
73}
74
75#[cfg(feature = "serde")]
76impl Serialize for PlFileDecryptionProperties {
77    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
78    where
79        S: serde::Serializer,
80    {
81        use serde::ser::Error;
82        Err(S::Error::custom(
83            "cannot serialize parquet decryption properties",
84        ))
85    }
86}
87
88#[cfg(feature = "dsl-schema")]
89impl schemars::JsonSchema for PlFileDecryptionProperties {
90    fn schema_name() -> std::borrow::Cow<'static, str> {
91        "PlFileDecryptionProperties".into()
92    }
93
94    fn schema_id() -> std::borrow::Cow<'static, str> {
95        std::borrow::Cow::Borrowed(concat!(module_path!(), "::", "PlFileDecryptionProperties"))
96    }
97
98    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
99        Vec::<u8>::json_schema(generator)
100    }
101}
102
103#[derive(Copy, Clone, Debug, Eq, PartialEq, Default, Hash)]
104#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
105#[cfg_attr(feature = "dsl-schema", derive(schemars::JsonSchema))]
106pub enum ParallelStrategy {
107    /// Don't parallelize
108    None,
109    /// Parallelize over the columns
110    Columns,
111    /// Parallelize over the row groups
112    RowGroups,
113    /// First evaluates the pushed-down predicates in parallel and determines a mask of which rows
114    /// to read. Then, it parallelizes over both the columns and the row groups while filtering out
115    /// rows that do not need to be read. This can provide significant speedups for large files
116    /// (i.e. many row-groups) with a predicate that filters clustered rows or filters heavily. In
117    /// other cases, this may slow down the scan compared other strategies.
118    ///
119    /// If no predicate is given, this falls back to back to [`ParallelStrategy::Auto`].
120    Prefiltered,
121    /// Automatically determine over which unit to parallelize
122    /// This will choose the most occurring unit.
123    #[default]
124    Auto,
125}
polars_io/parquet/read/options.rs

polars_io/parquet/read/
options.rs
