tysv

tysv allows streaming typed values from full text (String) or file streams (FileStream). This is particularly useful for large CSV files that cannot fit in the memory buffer.

gleam add tysv

Standard CSV

import gleam/float
import gleam/int
import gleam/string
import tysv
import tysv/writer
import tysv/string_writer as sw

const data = "Vanilla,23,0.5
Strawberry,24,0.75
Mango,34,1.75"

const rows = [
  #("Vanilla", 23, 0.5),
  #("Strawberry", 24, 0.75),
  #("Mango", 34, 1.75),
]

pub fn standard_csv() {
  // Stream typed values from CSV
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.from_text(data, [], list.prepend)
    |> list.reverse

  // Build CSV from streaming values
  let sw = text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> sw.new

  assert data == rows |> list.fold(sw, sw.write) |> sw.finish_as_string
}

Skip Header

// Dependencies and constants from the standard case are omitted for brevity.

pub fn skip_header() {
  let data = "Flavor,Price,Kg\n" <> data

  // Stream typed values from CSV (with header)
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.skip_header(True)
    |> tysv.from_text(data, [], list.prepend)
    |> list.reverse

  // Build CSV (with header) from streaming values
  let sw = text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> text.header(["Flavor", "Price", "Kg"])
    |> sw.new

  assert data == rows |> list.fold(sw, sw.write) |> sw.finish_as_string
}

Custom Separators

// Dependencies and constants from the standard case are omitted for brevity.

pub fn custom_separators() {
  let data =
    string.replace(data, ",", ";")
    |> string.replace("\n", "|")

  // Stream typed values from CSV (with custom separators)
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.col_sep(";")
    |> tysv.row_sep("|")
    |> tysv.from_text(data, [], list.prepend)
    |> list.reverse

  // Build CSV (with custom separators) from streaming values
  let sw = text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> text.col_sep(";")
    |> text.row_sep("|")
    |> sw.new

  assert data == rows |> list.fold(sw, sw.write) |> sw.finish_as_string
}

With remaining columns

pub fn standard_csv() {
  let data =
    data |> string.replace("\n", ",wibble,wobble\n") <> ",wibble,wobble"
  let rows = {
    use row <- list.map(rows)
    #(row, ["wibble", "wobble"])
  }

  // Stream typed values from CSV
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.from_text_with_rest(data, [], list.prepend)
    |> list.reverse
}

Further documentation can be found at https://hexdocs.pm/tysv.

Development

gleam test  # Run the tests

Provenance

All code in this project is written naturally. No AI generation or assistance is used to implement any functionality.

LICENSE

tysv is made available under the terms of the EUPL-1.2 License.