Random access binary files with mmap - drastically slows with big files

[danart]

Posted a question with code here: https://stackoverflow.com/questions/5662...-test-code

Here is the code (tested with python 3.5, requires 10 GB of storage):

import os, errno, sys
import random, time
import mmap



def create_binary_test_file():
	print("Creating files with 3,000,000,000 characters, takes a few seconds...")
	test_binary_file1 = open("test_binary_file1.testbin", "wb")
	test_binary_file2 = open("test_binary_file2.testbin", "wb")
	test_binary_file3 = open("test_binary_file3.testbin", "wb")
	for i in range(1000):
		if i % 100 == 0 :
			print("progress -  ", i/10, " % ")
		# efficiently create random strings and write to files
		tbl = bytes.maketrans(bytearray(range(256)),
	                      bytearray([ord(b'a') + b % 26 for b in range(256)]))
		random_string = (os.urandom(3000000).translate(tbl))
		test_binary_file1.write(str(random_string).encode('utf-8'))
		test_binary_file2.write(str(random_string).encode('utf-8'))
		test_binary_file3.write(str(random_string).encode('utf-8'))
	test_binary_file1.close()
	test_binary_file2.close()
	test_binary_file3.close()
	print("Created binary file for testing.The file contains 3,000,000,000 characters")




# Opening binary test file
try:
    binary_file = open("test_binary_file1.testbin", "r+b")
except OSError as e: # this would be "except OSError, e:" before Python 2.6
    if e.errno == errno.ENOENT: # errno.ENOENT = no such file or directory
    	create_binary_test_file()
    	binary_file = open("test_binary_file1.testbin", "r+b")




## example of use - perform 100 times, in each itteration: open one of the binary files and retrieve 5,000 sample strings
## (if code runs fast and without a slowdown - change k to 50000 and it should reproduce the problem)

## Example 1 - getting information from start of file
print("Getting information from start of file")
etime = []
for i in range(100):
	start = time.time()
	binary_file_mm = mmap.mmap(binary_file.fileno(), 0)
	sample_index_list = random.sample(range(1,100000-1000), k=50000)
	sampled_data = [[binary_file_mm[v:v+1000].decode("utf-8")] for v in sample_index_list]
	binary_file_mm.close()
	binary_file.close()
	file_number = random.randint(1, 3)
	binary_file = open("test_binary_file" + str(file_number) + ".testbin", "r+b")
	etime.append((time.time() - start))
	if i % 10 == 9 :
		print("Iter ", i, " \tAverage time - ", '%.5f' % (sum(etime[-9:]) / len(etime[-9:])))
binary_file.close()


## Example 2 - getting information from all of the file
print("Getting information from all of the file")
binary_file = open("test_binary_file1.testbin", "r+b")
etime = []
for i in range(100):
	start = time.time()
	binary_file_mm = mmap.mmap(binary_file.fileno(), 0)
	sample_index_list = random.sample(range(1,3000000000-1000), k=50000)
	sampled_data = [[binary_file_mm[v:v+1000].decode("utf-8")] for v in sample_index_list]
	binary_file_mm.close()
	binary_file.close()
	file_number = random.randint(1, 3)
	binary_file = open("test_binary_file" + str(file_number) + ".testbin", "r+b")
	etime.append((time.time() - start))
	if i % 10 == 9 :
		print("Iter ", i, " \tAverage time - ", '%.5f' % (sum(etime[-9:]) / len(etime[-9:])))
binary_file.close()