{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import hub\n", "import torch\n", "import numpy as np\n", "from hub.schema import Image, ClassLabel\n", "from hub.utils import Timer\n", "from torchvision import transforms\n", "import albumentations as A\n", "from albumentations.pytorch import ToTensor" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "ds = hub.load(\"activeloop/mnist\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Converting as is" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "mnist_torch = ds.to_pytorch()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['image', 'label'])\n", "torch.Size([28, 28, 1])\n" ] } ], "source": [ "print(mnist_torch[0].keys())\n", "print(mnist_torch[0]['image'].size())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Each sample of dataset, converted to pytorch with default parameters, is a dictionary with keys 'image' and 'label', shape of images is [28, 28, 1] -> H, W, C. Note, that this is not the order used in pytorch (should be C x H x W). We will try to output list and tuple" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[tensor([[[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 25],\n", " [ 43],\n", " [ 39],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 36],\n", " [227],\n", " [252],\n", " [247],\n", " [142],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 62],\n", " [233],\n", " [232],\n", " [135],\n", " [197],\n", " [241],\n", " [253],\n", " [252],\n", " [252],\n", " [247],\n", " [188],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 16],\n", " [221],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [226],\n", " [ 84],\n", " [ 84],\n", " [212],\n", " [252],\n", " [182],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 54],\n", " [186],\n", " [252],\n", " [253],\n", " [252],\n", " [155],\n", " [121],\n", " [ 24],\n", " [ 0],\n", " [ 0],\n", " [215],\n", " [252],\n", " [147],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [128],\n", " [253],\n", " [253],\n", " [194],\n", " [ 27],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [233],\n", " [253],\n", " [129],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 85],\n", " [168],\n", " [124],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [100],\n", " [247],\n", " [252],\n", " [ 42],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [190],\n", " [252],\n", " [185],\n", " [ 4],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 55],\n", " [232],\n", " [252],\n", " [ 84],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 45],\n", " [236],\n", " [252],\n", " [190],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 43],\n", " [123],\n", " [201],\n", " [253],\n", " [253],\n", " [255],\n", " [253],\n", " [200],\n", " [148],\n", " [236],\n", " [255],\n", " [239],\n", " [ 62],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 84],\n", " [190],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [ 89],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 82],\n", " [246],\n", " [252],\n", " [232],\n", " [231],\n", " [134],\n", " [126],\n", " [ 38],\n", " [ 21],\n", " [151],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [ 63],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 84],\n", " [246],\n", " [252],\n", " [155],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [126],\n", " [232],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [154],\n", " [ 6],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [190],\n", " [252],\n", " [155],\n", " [ 7],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 27],\n", " [150],\n", " [253],\n", " [252],\n", " [199],\n", " [ 77],\n", " [174],\n", " [253],\n", " [252],\n", " [109],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 61],\n", " [253],\n", " [214],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 32],\n", " [ 96],\n", " [218],\n", " [253],\n", " [247],\n", " [131],\n", " [ 53],\n", " [ 0],\n", " [ 0],\n", " [237],\n", " [253],\n", " [191],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [183],\n", " [252],\n", " [126],\n", " [ 0],\n", " [ 36],\n", " [103],\n", " [237],\n", " [252],\n", " [252],\n", " [182],\n", " [ 53],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 55],\n", " [231],\n", " [252],\n", " [163],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [253],\n", " [252],\n", " [189],\n", " [197],\n", " [241],\n", " [253],\n", " [245],\n", " [222],\n", " [ 56],\n", " [ 4],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [174],\n", " [251],\n", " [221],\n", " [ 74],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [156],\n", " [252],\n", " [252],\n", " [210],\n", " [145],\n", " [ 84],\n", " [ 56],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [161],\n", " [252],\n", " [121],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 7],\n", " [ 42],\n", " [ 42],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 21],\n", " [ 42],\n", " [ 7],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]]], dtype=torch.uint8), tensor(2)]\n" ] } ], "source": [ "mnist_torch = ds.to_pytorch(output_type=list)\n", "print(mnist_torch[0])" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([[[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 25],\n", " [ 43],\n", " [ 39],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 36],\n", " [227],\n", " [252],\n", " [247],\n", " [142],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 62],\n", " [233],\n", " [232],\n", " [135],\n", " [197],\n", " [241],\n", " [253],\n", " [252],\n", " [252],\n", " [247],\n", " [188],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 16],\n", " [221],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [226],\n", " [ 84],\n", " [ 84],\n", " [212],\n", " [252],\n", " [182],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 54],\n", " [186],\n", " [252],\n", " [253],\n", " [252],\n", " [155],\n", " [121],\n", " [ 24],\n", " [ 0],\n", " [ 0],\n", " [215],\n", " [252],\n", " [147],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [128],\n", " [253],\n", " [253],\n", " [194],\n", " [ 27],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [233],\n", " [253],\n", " [129],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 85],\n", " [168],\n", " [124],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [100],\n", " [247],\n", " [252],\n", " [ 42],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [190],\n", " [252],\n", " [185],\n", " [ 4],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 55],\n", " [232],\n", " [252],\n", " [ 84],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 45],\n", " [236],\n", " [252],\n", " [190],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 43],\n", " [123],\n", " [201],\n", " [253],\n", " [253],\n", " [255],\n", " [253],\n", " [200],\n", " [148],\n", " [236],\n", " [255],\n", " [239],\n", " [ 62],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 84],\n", " [190],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [252],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [ 89],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 82],\n", " [246],\n", " [252],\n", " [232],\n", " [231],\n", " [134],\n", " [126],\n", " [ 38],\n", " [ 21],\n", " [151],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [ 63],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 84],\n", " [246],\n", " [252],\n", " [155],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [126],\n", " [232],\n", " [252],\n", " [252],\n", " [252],\n", " [253],\n", " [154],\n", " [ 6],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [190],\n", " [252],\n", " [155],\n", " [ 7],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 27],\n", " [150],\n", " [253],\n", " [252],\n", " [199],\n", " [ 77],\n", " [174],\n", " [253],\n", " [252],\n", " [109],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 61],\n", " [253],\n", " [214],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 32],\n", " [ 96],\n", " [218],\n", " [253],\n", " [247],\n", " [131],\n", " [ 53],\n", " [ 0],\n", " [ 0],\n", " [237],\n", " [253],\n", " [191],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [183],\n", " [252],\n", " [126],\n", " [ 0],\n", " [ 36],\n", " [103],\n", " [237],\n", " [252],\n", " [252],\n", " [182],\n", " [ 53],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 55],\n", " [231],\n", " [252],\n", " [163],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [253],\n", " [252],\n", " [189],\n", " [197],\n", " [241],\n", " [253],\n", " [245],\n", " [222],\n", " [ 56],\n", " [ 4],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [174],\n", " [251],\n", " [221],\n", " [ 74],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [156],\n", " [252],\n", " [252],\n", " [210],\n", " [145],\n", " [ 84],\n", " [ 56],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [161],\n", " [252],\n", " [121],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 7],\n", " [ 42],\n", " [ 42],\n", " [ 14],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 21],\n", " [ 42],\n", " [ 7],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]],\n", "\n", " [[ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0],\n", " [ 0]]], dtype=torch.uint8), tensor(2))\n" ] } ], "source": [ "mnist_torch = ds.to_pytorch(output_type=tuple)\n", "print(mnist_torch[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Converting with Transforms - TorchVision" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's define torchvision transformations as a function. The function should work with initial dataset structure." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "torchvision_transform = transforms.Compose([\n", " transforms.ToPILImage(),\n", " transforms.Resize((256, 256)), \n", " transforms.RandomCrop(224),\n", " transforms.RandomHorizontalFlip(),\n", " transforms.ToTensor(),\n", " transforms.Normalize(\n", " mean=(0.1307,), # mean value for mnist dataset\n", " std=(0.3081,), # mean value for mnist dataset\n", " )\n", "])\n", "\n", "def transform(data):\n", " img = torchvision_transform(data['image'])\n", " label = torch.tensor(data['label'])\n", " return img, label" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We use `inplace=False` since we want to tranform original numpy arrays. Note, that now transform function defines output type" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([[[-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " ...,\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242]]]), tensor(2))\n" ] } ], "source": [ "mnist_torch = ds.to_pytorch(Transform=transform, inplace=False)\n", "\n", "print(mnist_torch[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Converting with Albumentations - TorchVision" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "albumentations_transform = A.Compose([\n", " A.Resize(256, 256), \n", " A.RandomCrop(224, 224),\n", " A.HorizontalFlip(),\n", " A.Normalize(\n", " mean=(0.1307,),\n", " std=(0.3081,),\n", " ),\n", " ToTensor()\n", "])" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def transform(data):\n", " augmented = albumentations_transform(image=data['image'])\n", " img = augmented['image']\n", " label = torch.tensor(data['label'])\n", " return img, label" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "mnist_torch = ds.to_pytorch(Transform=transform, inplace=False)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([[[-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " ...,\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242],\n", " [-0.4242, -0.4242, -0.4242, ..., -0.4242, -0.4242, -0.4242]]]), tensor(2))\n" ] } ], "source": [ "print(mnist_torch[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "SeedQualityPredictor", "language": "python", "name": "seedqualitypredictor" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 4 }