folder/train/dog/golden_retriever.png
folder/train/dog/german_shepherd.png
folder/train/dog/chihuahua.png

folder/train/cat/maine_coon.png
folder/train/cat/bengal.png
folder/train/cat/birman.png

>>> from datasets import load_dataset

>>> dataset = load_dataset("path/to/folder")

>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder")

folder/train/dog/golden_retriever.png
folder/train/cat/maine_coon.png
folder/test/dog/german_shepherd.png
folder/test/cat/bengal.png

folder/train/metadata.csv
folder/train/0001.png
folder/train/0002.png
folder/train/0003.png

folder/train.zip
folder/test.zip
folder/validation.zip

file_name,additional_feature
0001.png,This is a first value of a text feature you added to your images
0002.png,This is a second value of a text feature you added to your images
0003.png,This is a third value of a text feature you added to your images

{"file_name": "0001.png", "additional_feature": "This is a first value of a text feature you added to your images"}
{"file_name": "0002.png", "additional_feature": "This is a second value of a text feature you added to your images"}
{"file_name": "0003.png", "additional_feature": "This is a third value of a text feature you added to your images"}

{"input_file_name": "0001.png", "output_file_name": "0001_output.png"}
{"input_file_name": "0002.png", "output_file_name": "0002_output.png"}
{"input_file_name": "0003.png", "output_file_name": "0003_output.png"}

{"frames_file_names": ["0001_t0.png", "0001_t1.png"], label: "moving_up"}
{"frames_file_names": ["0002_t0.png", "0002_t1.png"], label: "moving_down"}
{"frames_file_names": ["0003_t0.png", "0003_t1.png"], label: "moving_right"}

file_name,text
0001.png,This is a golden retriever playing with a ball
0002.png,A german shepherd
0003.png,One chihuahua

>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
>>> dataset[0]["text"]
"This is a golden retriever playing with a ball"

{"file_name": "0001.png", "objects": {"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}}
{"file_name": "0002.png", "objects": {"bbox": [[810.0, 100.0, 57.0, 28.0]], "categories": [1]}}
{"file_name": "0003.png", "objects": {"bbox": [[160.0, 31.0, 248.0, 616.0], [741.0, 68.0, 202.0, 401.0]], "categories": [2, 2]}}

>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
>>> dataset[0]["objects"]
{"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}

>>> from datasets import load_dataset

>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
>>> dataset.push_to_hub("stevhliu/my-image-captioning-dataset")

folder/train/00000.tar
folder/train/00001.tar
folder/train/00002.tar
...

e39871fd9fd74f55.jpg
e39871fd9fd74f55.json
f18b91585c4d3f3e.jpg
f18b91585c4d3f3e.json
ede6e66b2fb59aab.jpg
ede6e66b2fb59aab.json
ed600d57fcee4f94.jpg
ed600d57fcee4f94.json
...

>>> from datasets import load_dataset

>>> dataset = load_dataset("webdataset", data_dir="/path/to/folder", split="train")
>>> dataset[0]["json"]
{"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}

e39871fd9fd74f55.input.jpg
e39871fd9fd74f55.output.jpg
e39871fd9fd74f55.json
f18b91585c4d3f3e.input.jpg
f18b91585c4d3f3e.output.jpg
f18b91585c4d3f3e.json
...

{'caption': 'Cordelia and Dudley on their wedding  day last year', 'height': 315, 'width': 233}
{'caption': 'Statistics on challenges for automation in 2021', 'height': 299, 'width': 701}

import lance
import pyarrow as pa

schema = pa.schema(
    [
        pa.field("caption", pa.utf8()),
        pa.field("height", pa.int32()),
        pa.field("width", pa.int32()),
        # ... add any additional metadata columns you want here ...
        pa.field("image", pa.binary()),
    ]
)

# Provide image files alongside metadata
rows = [
    {
        "image_path": "/path/to/images/0001.jpg",
        "caption": "Cordelia and Dudley on their wedding  day last year",
        "height": 315,
        "width": 233,
    },
    {
        "image_path": "/path/to/images/0002.jpg",
        "caption": "Statistics on challenges for automation in 2021",
        "height": 299,
        "width": 701,
    },
]

image_bytes = []
for r in rows:
    with open(r["image_path"], "rb") as f:
        image_bytes.append(f.read())

table = pa.table(
    {
        "caption": [r["caption"] for r in rows],
        "height": [r["height"] for r in rows],
        "width": [r["width"] for r in rows],
        "image": image_bytes,
    },
    schema=schema,
)

ds = lance.write_dataset(
    table,
    "./images.lance",
    schema=schema,
    mode="create",
)

+-----------------------------------------------+--------+-------+-----+------------------------------+
| caption                                       | height | width | ... | image                        |
+-----------------------------------------------+--------+-------+-----+------------------------------+
| "Cordelia and Dudley on their wedding ..."    | 315    | 233   | ... | b"\\xff\\xd8\\xff...\\xd9"   |
| "Statistics on challenges for automation ..." | 299    | 701   | ... | b"\\xff\\xd8\\xff...\\xd9"   |
+-----------------------------------------------+--------+-------+-----+------------------------------+