1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

use arrow::io::ipc::write::file_async::FileSink;
use arrow::io::ipc::write::WriteOptions;
use futures::{AsyncWrite, SinkExt};
use polars_core::prelude::*;

use crate::ipc::IpcWriter;

impl<W: AsyncWrite + Unpin + Send> IpcWriter<W> {
    pub fn new_async(writer: W) -> Self {
        IpcWriter {
            writer,
            compression: None,
            pl_flavor: false,
        }
    }

    pub fn batched_async(self, schema: &Schema) -> PolarsResult<BatchedWriterAsync<W>> {
        let writer = FileSink::new(
            self.writer,
            schema.to_arrow(false),
            None,
            WriteOptions {
                compression: self.compression.map(|c| c.into()),
            },
        );

        Ok(BatchedWriterAsync { writer })
    }
}

pub struct BatchedWriterAsync<'a, W>
where
    W: AsyncWrite + Unpin + Send + 'a,
{
    writer: FileSink<'a, W>,
}

impl<'a, W> BatchedWriterAsync<'a, W>
where
    W: AsyncWrite + Unpin + Send + 'a,
{
    /// Write a batch to the parquet writer.
    ///
    /// # Panics
    /// The caller must ensure the chunks in the given [`DataFrame`] are aligned.
    pub async fn write_batch(&mut self, df: &DataFrame) -> PolarsResult<()> {
        let iter = df.iter_chunks(false);
        for batch in iter {
            self.writer.feed(batch.into()).await?;
        }
        Ok(())
    }

    /// Writes the footer of the IPC file.
    pub async fn finish(&mut self) -> PolarsResult<()> {
        self.writer.close().await?;
        Ok(())
    }
}