LMDB数据库加速Pytorch文件读取速度

 1 import lmdb
 2 import os, sys
 3 
 4 def initialize(lmdb_dir, map_size):
 5     # map_size: bytes
 6     env = lmdb.open(lmdb_dir, map_size)
 7     return env
 8 
 9 def insert(env, key, value):
10     txn = env.begin(write=True)
11     txn.put(str(key).encode(), value.encode())
12     txn.commit()
13 
14 def delete(env, key):
15     txn = env.begin(write=True)
16     txn.delete(str(key).encode())
17     txn.commit()
18 
19 def update(env, key, value):
20     txn = env.begin(write=True)
21     txn.put(str(key).encode(), value.encode())
22     txn.commit()
23 
24 def search(env, key):
25     txn = env.begin()
26     value = txn.get(str(key).encode())
27     return value
28 
29 def display(env):
30     txn = env.begin()
31     cursor = txn.cursor()
32     for key, value in cursor:
33         print(key, value)
34 
35 
36 if __name__ == ‘__main__‘:
37     path = ‘D:/desktop/lmdb‘
38     env = initialize(path, 10*1024*1024)
39 
40     print("Insert 3 records.")
41     insert(env, 1, "Alice")
42     insert(env, 2, "Bob")
43     insert(env, 3, "Peter")
44     display(env)
45 
46     print("Delete the record where key = 1")
47     delete(env, 1)
48     display(env)
49 
50     print("Update the record where key = 3")
51     update(env, 3, "Mark")
52     display(env)
53 
54     print("Get the value whose key = 3")
55     name = search(env, 3)
56     print(name)
57 
58     # 最后需要关闭lmdb数据库
59     env.close()

 1 import lmdb
 2 
 3 image_path = ‘./cat.jpg‘
 4 label = ‘cat‘
 5 
 6 env = lmdb.open(‘lmdb_dir‘)
 7 cache = {}  # 存储键值对
 8 
 9 with open(image_path, ‘rb‘) as f:
10     # 读取图像文件的二进制格式数据
11     image_bin = f.read()
12 
13 # 用两个键值对表示一个数据样本
14 cache[‘image_000‘] = image_bin
15 cache[‘label_000‘] = label
16 
17 with env.begin(write=True) as txn:
18     for k, v in cache.items():
19         if isinstance(v, bytes):
20             # 图片类型为bytes
21             txn.put(k.encode(), v)
22         else:
23             # 标签类型为str, 转为bytes
24             txn.put(k.encode(), v.encode())  # 编码
25 
26 env.close()

 1 import cv2
 2 import lmdb
 3 import numpy as np
 4 
 5 env = lmdb.open(‘lmdb_dir‘)
 6 
 7 with env.begin(write=False) as txn:
 8     # 获取图像数据
 9     image_bin = txn.get(‘image_000‘.encode())
10     label = txn.get(‘label_000‘.encode()).decode()  # 解码
11 
12     # 将二进制文件转为十进制文件（一维数组）
13     image_buf = np.frombuffer(image_bin, dtype=np.uint8)
14     # 将数据转换(解码)成图像格式
15     # cv2.IMREAD_GRAYSCALE为灰度图，cv2.IMREAD_COLOR为彩色图
16     img = cv2.imdecode(image_buf, cv2.IMREAD_COLOR)
17     cv2.imshow(‘image‘, img)
18     cv2.waitKey(0)

 1 #!/usr/bin/env python
 2 
 3 import leveldb
 4 import os, sys
 5 
 6 def initialize():
 7     db = leveldb.LevelDB("students");
 8     return db;
 9 
10 def insert(db, sid, name):
11     db.Put(str(sid), name);
12 
13 def delete(db, sid):
14     db.Delete(str(sid));
15 
16 def update(db, sid, name):
17     db.Put(str(sid), name);
18 
19 def search(db, sid):
20     name = db.Get(str(sid));
21     return name;
22 
23 def display(db):
24     for key, value in db.RangeIter():
25         print (key, value);
26 
27 db = initialize();
28 
29 print "Insert 3 records."
30 insert(db, 1, "Alice");
31 insert(db, 2, "Bob");
32 insert(db, 3, "Peter");
33 display(db);
34 
35 print "Delete the record where sid = 1."
36 delete(db, 1);
37 display(db);
38 
39 print "Update the record where sid = 3."
40 update(db, 3, "Mark");
41 display(db);
42 
43 print "Get the name of student whose sid = 3."
44 name = search(db, 3);
45 print name;