diff --git a/examples/aistore/README.md b/examples/aistore/README.md new file mode 100644 index 0000000..4cecbd4 --- /dev/null +++ b/examples/aistore/README.md @@ -0,0 +1,14 @@ +## Run Hub on AIStore + +Start a production-ready standalone docker of aistore. More information can be found [here](https://nvidia.github.io/aistore/deploy/prod/docker/single). +```sh +docker run \ + -p 51080:51080 \ + -v $(mktemp -d):/ais/disk0 \ + aistore/cluster-minimal:latest +``` + +Run the script +``` +python3 examples/aistore/simple.py +``` \ No newline at end of file diff --git a/examples/aistore/simple.py b/examples/aistore/simple.py new file mode 100644 index 0000000..e84a35d --- /dev/null +++ b/examples/aistore/simple.py @@ -0,0 +1,30 @@ +import hub +import numpy as np + +creds = {'endpoint_url': 'http://localhost:51080/s3'} + + +def create_dataset(path: str): + """ Create hub dataset and upload random data """ + + ds = hub.empty(path, creds=creds, overwrite=True) + with ds: + ds.create_tensor("tensor") + for i in range(10): + ds.tensor.append(np.random.random((512, 512))) + + +def loop(path: str): + """ Load the dataset and stream to pytorch""" + ds = hub.load(path, creds) + dataloader = ds.pytorch() + + for (x,) in dataloader: + print(x) + + +if __name__ == "__main__": + path = 's3://ubuntu/dataset' + + create_dataset(path) + loop(path)