Python Forum

datas  = np.zeros((819200, 460800, 4), np.unint8)

Error:
Unable to allocate 1.37 TiB for an array with shape (819200, 460800, 4) and data type uint8

with h5py.File("mytestfile.hdf5", "w") as f:
	dset = f.create_dataset("mydataset", data=(819200, 460800, 4), dtype='i', compression='gzip') #creates 4KB file

data = np.memmap('output.dat', shape=(819200, 460800, 4), mode='w+', dtype='i')

(Jun-04-2020, 03:57 AM)scidam Wrote: [ -> ]Numpy allows to create arrays that are stored on disk. Take a look at memmap function.

data = np.memmap('output.dat', shape=(819200, 460800, 4), mode='w+', dtype='i')

def factors(x):
    result = []
    i = 1
    while i*i <= x:
        if x % i == 0:
            result.append(i)
            if x//i != i:
                result.append(x//i)
        i += 1
    return result
def get_divisor(x):
	fctrs = sorted(factors(x[0]))[::-1]
	i = 0
	while True:
		try:
			a = np.zeros((fctrs[i], fctrs[i], 4)) #finding the largest array that can be stored in memory means I can find the highest factor that will allow me to split up the original array and keep it as large as possible
			return fctrs[i]
		except MemoryError:
			pass
		i += 1

def test():
	f = h5py.File("mytestfile.hdf5", "w")
	dset = f.create_dataset("mydataset", (100000,100000,4), dtype=np.uint8, compression='gzip')

	#arr = np.random.uniform(low=0, high=255, size=(10000,10000,4))
	shp = dset.shape	
	step = get_divisor(shp)


	png = open("new.png", "wb")
	numpngw._write_header_and_meta(png, 8, shp, color_type=6, bitdepth=8, palette=None, #i'm manually writing to the png file rather than writing all data at once, so i can append data over and over again.
							interlace=0, text_list=None, timestamp=None, sbit=None, gamma=None, iccp=None,
							chromaticity=None, trans=None, background=None, phys=None)

	#_max = shp[0] if not step == shp[0] else shp[0]*2 #range is 'upto but not including' so if for some reason 'step' equals the size of the array, I 'double the size' to allow step to reach the real size of the array.
	for i in range(step, shp[0]+step, step): #from step to 
		numpngw._write_data(png, dset[:i], bitdepth=8, max_chunk_len=step, #writing the data in largest chunks I can
					filter_type=None, interlace=0)
		png.flush() #flush the buffer
		gc.collect() #try and free up memory for next round.

	numpngw._write_iend(png)

	png.close()
	f.close()

f = h5py.File("test.hdf5", "w")
dset = f.create_dataset("test", (10000,10000,3), dtype=np.uint8, compression='gzip')

memmap_image = tifffile.memmap('temp.tif', shape=dset.shape, dtype='uint8')
memmap_image[:] = dset[:]
memmap_image.flush()
del memmap_image
gc.collect()

(Jun-06-2020, 01:34 PM)DreamingInsanity Wrote: [ -> ]Don't even know if anyone is going to ever need this answer but it's here just in case.
I tried out PIL again. I decided to use TIFF files (they are fast, and can be animated so I don't need another library for gifs). Unfortunately I had the same problem I've had with all the libraries where they don't support appending data to an image (ie. you cant open the image file in the mode 'a'). Although Image.save have the parameter 'append' for TIFF files, that's used ig you want to add more than one image to the TIFF file.
I stuck with TIFFs and looked for a good python TIFF library. tifffile seemed to do everything I needed. It even has a memmap implementation which I used.
My code is now:

f = h5py.File("test.hdf5", "w")
dset = f.create_dataset("test", (10000,10000,3), dtype=np.uint8, compression='gzip')

memmap_image = tifffile.memmap('temp.tif', shape=dset.shape, dtype='uint8')
memmap_image[:] = dset[:]
memmap_image.flush()
del memmap_image
gc.collect()

I'm sure there's error that's going to crop up somehow somewhere in my code but at least it's working for now.

1. a basic image is created using matplotlib
   
2. the image is saved into the h5 file as a picture (you can see it under hdfview for instance)
   
3. the image is saved into the h5 file as an array
   
4. the previous picture is recovered an saved locally into a gif image
   

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvas
import os, h5py
 
Path = str(os.getcwd())

### 0) to create a fig
plt.style.use('_mpl-gallery')

# make data
x = np.linspace(0, 10, 100)
y = 4 + 2 * np.sin(2 * x)

# plot
fig, ax = plt.subplots(figsize=(16,16))
ax.plot(x, y, linewidth=2.0)

ax.set(xlim=(0, 8), xticks=np.arange(1, 8),
       ylim=(0, 8), yticks=np.arange(1, 8))

plt.show()
 
 
## 1) Convertion  Matplotlib fig into aNumpyArray ---> mandatory to change the picture into a matrix      
canvas = FigureCanvas(fig)
canvas.draw()
FigArray = np.array(canvas.renderer.buffer_rgba())
 
## 2) write directly a picture into the h5 file --> Note dataset is MANDATORY
h5 = h5py.File(Path + '/test_picture.h5', 'w')
ImageDataset = h5.create_dataset(name = "Example_picture", data = FigArray, dtype = 'uint8', compression = 'gzip')
ImageDataset.attrs["CLASS"] = np.string_("IMAGE")
ImageDataset.attrs["IMAGE_VERSION"] = np.string_("1.2")
ImageDataset.attrs["IMAGE_SUBCLASS"] = np.string_("IMAGE_TRUECOLOR")
ImageDataset.attrs["INTERLACE_MODE"] = np.string_("INTERLACE_MODE")
ImageDataset.attrs["IMAGE_MINMAXRANGE"] = np.uint8(0.255)
h5.close()


## 3) Write the fig into a basic array (the h5 file)
h5 = h5py.File(Path + '/test_array.h5', 'w')
NpArrays = h5.create_group('Fig')
DatasetFigs = NpArrays.create_dataset(name = 'Example of array', data = FigArray, dtype='f', compression = 'gzip')
h5.flush()
h5.close()


## 4) array recovering => save into a gif file
from PIL import Image as im
with h5py.File(Path + '/test_picture.h5','r') as pict:
    data = pict.get('/Example_picture')
    Array = np.array(data)
    
    # array is converted int a picture
    Picture = im.fromarray(Array)
    Picture.save(Path + '/export.gif')