kd.data.py.Json

class kauldron.data.py.Json(path: str, *, _fake_refs: type[_FakeRefsUnset] | dict[str, _FakeRootCfg] = <class 'kauldron.utils.config_util._FakeRefsUnset'>, batch_size: int | None = None, seed: int | collections.abc.Sequence[int] | numpy.ndarray | jaxtyping.UInt32[Array, '2'] | jaxtyping.UInt32[ndarray, '2'] | jax.Array | None = _FakeRootCfg('cfg.seed'), transforms: tr_normalize.Transformations = <factory>, num_epochs: Optional[int] = None, batch_drop_remainder: bool = True, num_workers: int = 16, read_options: grain.ReadOptions | None = None, enable_profiling: bool = False, per_worker_buffer_size: int = 1, worker_init_fn: Callable[[int, int], None] | None = None, shuffle: bool)

Bases: kauldron.data.py.base.DataSourceBase

Json pipeline.

Assumes that the json file is a list of examples. The file will be loaded and kept in memory.

path: str

property data_source: grain._src.python.data_sources.RandomAccessDataSource