(Aug-16-2019, 08:52 PM)Gribouillis Wrote: I managed to get a similar performance in python 3 by using a temporary list to store a row of pixels. It seems that the problem indeed comes from the repeated concatenation of a single pixel to a bytes string. Here is my faster code.
# Testing struct.pack and string catenation in Python2 and 3 # This is a demo cut down from real app (which draws charts from survey data) # creates a 'square rainbow' bmp file ## edit for Py 2 (char strings) or 3 (byte strings) versions # edit these for your set up and test Size = 1024 # test image size, pixels # path = 'D:/Python37/MyScripts/Test/' # for the bmp file import csv import os import struct from math import trunc, ceil, floor import time path = os.path.join(os.path.dirname(__file__), 'test', '') def BuildImage(name, XY): # name : filename # XY : (width, height) pixels # for stats and timing n0 = 0 t00 = time.clock() t01 = t00 chtName = path+'cht_'+name+'3.bmp' print("drawing "+chtName) hdr = bmpHdr(XY) #print(hdr) ##pixels ='' ## Py2 pixels = bytes('', 'utf-8') ## Py3 for Y in range(0, XY[1]): # (BMPs are L to R from the bottom L row) temp = [] for X in range(0, XY[0]): # square rainbow for time tests - as oposed to real data x = floor((255 * X)/XY[0]) y = floor((255 * Y)/XY[1]) (r,g,b) = [x, y, 128] #Colour(data[x ,y]) temp.append(struct.pack('<BBB',b,g,r)) pixels += b''.join(temp) row_mod = (hdr['width']*hdr['colordepth']/8) % 4 if row_mod == 0: padding = 0 else: padding = (4 - row_mod) ##padbytes = '' # P2 padbytes = bytes('', 'utf-8') # P3 for i in range(padding): padbytes += struct.pack('<B',0) pixels = pixels + padbytes # stats log if(0 == Y % 100 or Y == 0): n = len(pixels) t02 = time.clock() log = "{0:5d} L={1:8,d}, delta={2:7,d}, pad={3:4d}".format(XY[0]-Y, n, n-n0, padding) log += ", time = {0:6.3f}, cum = {1:7.3f}".format(t02-t01, t02-t00) print(log) t01 = t02 n0 = n print("pixels generated, len = "+str(len(pixels))) bmp_write(chtName, hdr, pixels) def bmpHdr(XY): print("bmphdr xy "+str(XY)) hdr = { 'mn1':66, 'mn2':77, 'filesize':0, 'undef1':0, 'undef2':0, 'offset':54, 'headerlength':40, 'width':XY[0], #256 'height':XY[1], #256 'colorplanes':0, 'colordepth':24, 'compression':0, 'imagesize':0, 'res_hor':0, 'res_vert':0, 'palette':0, 'importantcolors':0 } return hdr #Function to write a bmp file. It takes a dictionary (hdr) of #header values and the pixel data (pixels) and writes them #to a file. This function is called at the bottom of the code. def bmp_write(name, hdr, pixels): print('making bmp with '+str(len(pixels))+" pixels") mn1 = struct.pack('<B',hdr['mn1']) mn2 = struct.pack('<B',hdr['mn2']) filesize = struct.pack('<L',hdr['filesize']) undef1 = struct.pack('<H',hdr['undef1']) undef2 = struct.pack('<H',hdr['undef2']) offset = struct.pack('<L',hdr['offset']) headerlength = struct.pack('<L',hdr['headerlength']) width = struct.pack('<L',hdr['width']) height = struct.pack('<L',hdr['height']) colorplanes = struct.pack('<H',hdr['colorplanes']) colordepth = struct.pack('<H',hdr['colordepth']) compression = struct.pack('<L',hdr['compression']) imagesize = struct.pack('<L',hdr['imagesize']) res_hor = struct.pack('<L',hdr['res_hor']) res_vert = struct.pack('<L',hdr['res_vert']) palette = struct.pack('<L',hdr['palette']) importantcolors = struct.pack('<L',hdr['importantcolors']) #create the outfile outfile = open(name,'wb') # 'bitmap_image.bmp' #write the header + the_bytes hdr = mn1+mn2 hdr += filesize+undef1+undef2 hdr += offset+headerlength+width+height hdr += colorplanes+colordepth+compression+imagesize+res_hor+res_vert hdr += palette+importantcolors print("headers = "+str(hdr)) bmp = hdr + pixels print('writing bmp, len = '+str(len(bmp))) outfile.write(bmp) ################################### def main(): time0 = time.clock() print("start {0}x{0} bmp file @ {1:.3f}".format(Size, time0)) # set the size of the bmp image here BuildImage("test", (Size,Size)) time1 = time.clock() print("Chart complete, run time {0:.3f} secs".format(time1-time0)) if __name__ == '__main__': main()
Good observation. You get just about the same speed up using a string for the row array (as opposed to your list), the key is working row-wise as it were, I think.
(Aug-17-2019, 07:21 AM)Gribouillis Wrote: I've found this reference. It seems that you could replace bytes with bytearray for better performance. The problem is also known to David Beazley, see here.
What has me confused is that reason given for the poor performance is immutability (bytearrays are mutable).
However bytes and string objects are both immutable, so that in and of itself is not the reason.