trunk/TELTest/BasicKeyValue.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace TELTest {
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;

    internal class BasicKeyValue {
        public BasicKeyValue(string directoryName) {
            DirectoryName = directoryName;
            LoadAllKeys();
        }

        public long MaximumMemoryPressure { get; set; } = 2L * 1024 * 1024 * 1024; // 2 GB
        public long CurrentMemoryPressure => m_CurrentMemoryPressure;
        public string DirectoryName { get; set; } = "Database";
        public int LastTuplesOut { get; protected set; }
        private long m_CurrentMemoryPressure = 0;
        private readonly object _dmpLock = new object();

        private readonly Dictionary<char, DateTime> m_LastAccess = new Dictionary<char, DateTime>();
        private readonly HashSet<string> m_AllKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        private readonly Dictionary<char, long> m_ChunkMemoryPressure = new Dictionary<char, long>();
        private readonly Dictionary<char, Dictionary<string, byte[]>> m_Chunks = new Dictionary<char, Dictionary<string, byte[]>>();

        public bool ContainsKey(string key) => m_AllKeys.Contains(key);

        public byte[] this[string key] {
            get {
                char q = char.ToUpper(key[0]);

                if (!m_AllKeys.Contains(key))
                    return null;

                if (!m_Chunks.ContainsKey(q) || !m_Chunks[q].ContainsKey(key)) {
                    LoadAndFreeIfNeeded(q);
                }

                if (m_Chunks.ContainsKey(q) && m_Chunks[q].ContainsKey(key)) {
                    m_LastAccess[q] = DateTime.Now;
                    return m_Chunks[q][key];
                }

                return null;
            }
            set => AddOrSet(key, value);
        }

        public void Remove(string key) {
            char c = char.ToUpper(key[0]);
            if (!m_AllKeys.Remove(key)) return;

            if (m_Chunks.ContainsKey(c)) {
                var oldValue = m_Chunks[c][key];
                m_Chunks[c].Remove(key);
                m_ChunkMemoryPressure[c] -= oldValue.Length;
                m_CurrentMemoryPressure -= oldValue.Length;
            } else {
                LoadAndFreeIfNeeded(c);
                if (m_Chunks[c].ContainsKey(key)) {
                    var loadedValue = m_Chunks[c][key];
                    if (m_Chunks[c].Remove(key)) {
                        m_ChunkMemoryPressure[c] -= loadedValue.Length;
                        m_CurrentMemoryPressure -= loadedValue.Length;
                    }
                }
            }
        }

        public void SavePendingChanges() => DumpAll();

        protected void AddOrSet(string key, byte[] value) {
            char q = char.ToUpper(key[0]);

            PrepAddChunk(q);

            if (m_Chunks[q].TryGetValue(key, out var oldValue)) {
                m_Chunks[q][key] = value;
                long delta = value.Length - oldValue.Length;
                m_ChunkMemoryPressure[q] += delta;
                m_CurrentMemoryPressure += delta;
            } else {
                m_AllKeys.Add(key);
                m_Chunks[q].Add(key, value);
                m_ChunkMemoryPressure[q] += value.Length;
                m_CurrentMemoryPressure += value.Length;
            }
        }

        protected void LoadAndFreeIfNeeded(char at) {
            while (m_CurrentMemoryPressure > MaximumMemoryPressure && m_LastAccess.Any()) {
                var oldest = m_LastAccess.OrderBy(x => x.Value).First();
                UnloadChunk(oldest.Key);
            }
            LoadChunk(at);
        }

        protected void UnloadChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!m_Chunks.ContainsKey(chk)) return;

            WriteChunk(chk);

            m_CurrentMemoryPressure -= m_ChunkMemoryPressure[chk];
            m_ChunkMemoryPressure.Remove(chk);

            m_Chunks.Remove(chk);
            m_LastAccess.Remove(chk);
        }

        public void DumpAll() {
            LastTuplesOut = 0;
            lock (_dmpLock) {
                if (!Directory.Exists(DirectoryName))
                    Directory.CreateDirectory(DirectoryName);

                foreach (var item in m_Chunks.Keys.ToList()) {
                    WriteChunk(item);
                }
            }
        }

        protected void WriteChunk(char c) {
            c = char.ToUpper(c);
            if (!m_Chunks.ContainsKey(c)) return;

            string fp = Path.Combine(DirectoryName, c + ".bin");

            using (var str = File.Open(fp, FileMode.Create, FileAccess.Write))
            using (var bw = new BinaryWriter(str, System.Text.Encoding.UTF8, false)) {
                foreach (var kv in m_Chunks[c]) {
                    byte[] idBytes = System.Text.Encoding.UTF8.GetBytes(kv.Key);
                    bw.Write(idBytes.Length);
                    bw.Write(idBytes);

                    byte[] valueBytes = kv.Value ?? new byte[0];
                    bw.Write(valueBytes.Length);
                    bw.Write(valueBytes);
                    LastTuplesOut++;
                }
                bw.Write(0); // terminator
            }
        }

        protected void LoadAllKeys() {
            if (!Directory.Exists(DirectoryName)) return;

            foreach (var chunk in Directory.GetFiles(DirectoryName, "*.bin")) {
                var cn = Path.GetFileNameWithoutExtension(chunk)[0];
                m_LastAccess[cn] = DateTime.MinValue;

                using (var fs = File.OpenRead(chunk))
                using (var br = new BinaryReader(fs, System.Text.Encoding.UTF8, true)) {
                    while (true) {
                        int idlen = br.ReadInt32();
                        if (idlen == 0) break;

                        string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idlen));
                        int ctlen = br.ReadInt32();
                        fs.Seek(ctlen, SeekOrigin.Current);

                        m_AllKeys.Add(id);
                    }
                }
            }
        }

        protected void PrepAddChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!m_Chunks.ContainsKey(chk))
                m_Chunks[chk] = new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);

            if (!m_ChunkMemoryPressure.ContainsKey(chk))
                m_ChunkMemoryPressure[chk] = 0L;

            m_LastAccess[chk] = DateTime.Now;
        }

        protected bool LoadChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!Directory.Exists(DirectoryName)) return false;

            string path = Path.Combine(DirectoryName, chk + ".bin");
            if (!File.Exists(path)) return false;
            if (m_Chunks.ContainsKey(chk)) return true;

            PrepAddChunk(chk);

            using (var str = File.OpenRead(path))
            using (var br = new BinaryReader(str, System.Text.Encoding.UTF8, true)) {
                while (true) {
                    int idl = br.ReadInt32();
                    if (idl == 0) break;

                    string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idl));
                    int ctlen = br.ReadInt32();
                    byte[] ctbytes = br.ReadBytes(ctlen);

                    m_Chunks[chk][id] = ctbytes;
                    m_AllKeys.Add(id);

                    long mp = id.Length + ctbytes.Length;
                    m_CurrentMemoryPressure += mp;
                    m_ChunkMemoryPressure[chk] += mp;
                }
            }

            return true;
        }
    }

}
Revision:	2
Committed:	Wed Feb 11 13:07:01 2026 UTC (8 weeks, 3 days ago) by figdor32
File size:	7999 byte(s)
Log Message:	Initial check-in
#	Content
1	using System;
2	using System.Collections.Generic;
3	using System.IO;
4	using System.Linq;
5	using System.Text;
6	using System.Threading.Tasks;
7
8	namespace TELTest {
9	using System;
10	using System.Collections.Generic;
11	using System.IO;
12	using System.Linq;
13
14	internal class BasicKeyValue {
15	public BasicKeyValue(string directoryName) {
16	DirectoryName = directoryName;
17	LoadAllKeys();
18	}
19
20	public long MaximumMemoryPressure { get; set; } = 2L * 1024 * 1024 * 1024; // 2 GB
21	public long CurrentMemoryPressure => m_CurrentMemoryPressure;
22	public string DirectoryName { get; set; } = "Database";
23	public int LastTuplesOut { get; protected set; }
24	private long m_CurrentMemoryPressure = 0;
25	private readonly object _dmpLock = new object();
26
27	private readonly Dictionary<char, DateTime> m_LastAccess = new Dictionary<char, DateTime>();
28	private readonly HashSet<string> m_AllKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
29	private readonly Dictionary<char, long> m_ChunkMemoryPressure = new Dictionary<char, long>();
30	private readonly Dictionary<char, Dictionary<string, byte[]>> m_Chunks = new Dictionary<char, Dictionary<string, byte[]>>();
31
32	public bool ContainsKey(string key) => m_AllKeys.Contains(key);
33
34	public byte[] this[string key] {
35	get {
36	char q = char.ToUpper(key[0]);
37
38	if (!m_AllKeys.Contains(key))
39	return null;
40
41	if (!m_Chunks.ContainsKey(q) \|\| !m_Chunks[q].ContainsKey(key)) {
42	LoadAndFreeIfNeeded(q);
43	}
44
45	if (m_Chunks.ContainsKey(q) && m_Chunks[q].ContainsKey(key)) {
46	m_LastAccess[q] = DateTime.Now;
47	return m_Chunks[q][key];
48	}
49
50	return null;
51	}
52	set => AddOrSet(key, value);
53	}
54
55	public void Remove(string key) {
56	char c = char.ToUpper(key[0]);
57	if (!m_AllKeys.Remove(key)) return;
58
59	if (m_Chunks.ContainsKey(c)) {
60	var oldValue = m_Chunks[c][key];
61	m_Chunks[c].Remove(key);
62	m_ChunkMemoryPressure[c] -= oldValue.Length;
63	m_CurrentMemoryPressure -= oldValue.Length;
64	} else {
65	LoadAndFreeIfNeeded(c);
66	if (m_Chunks[c].ContainsKey(key)) {
67	var loadedValue = m_Chunks[c][key];
68	if (m_Chunks[c].Remove(key)) {
69	m_ChunkMemoryPressure[c] -= loadedValue.Length;
70	m_CurrentMemoryPressure -= loadedValue.Length;
71	}
72	}
73	}
74	}
75
76	public void SavePendingChanges() => DumpAll();
77
78	protected void AddOrSet(string key, byte[] value) {
79	char q = char.ToUpper(key[0]);
80
81	PrepAddChunk(q);
82
83	if (m_Chunks[q].TryGetValue(key, out var oldValue)) {
84	m_Chunks[q][key] = value;
85	long delta = value.Length - oldValue.Length;
86	m_ChunkMemoryPressure[q] += delta;
87	m_CurrentMemoryPressure += delta;
88	} else {
89	m_AllKeys.Add(key);
90	m_Chunks[q].Add(key, value);
91	m_ChunkMemoryPressure[q] += value.Length;
92	m_CurrentMemoryPressure += value.Length;
93	}
94	}
95
96	protected void LoadAndFreeIfNeeded(char at) {
97	while (m_CurrentMemoryPressure > MaximumMemoryPressure && m_LastAccess.Any()) {
98	var oldest = m_LastAccess.OrderBy(x => x.Value).First();
99	UnloadChunk(oldest.Key);
100	}
101	LoadChunk(at);
102	}
103
104	protected void UnloadChunk(char chk) {
105	chk = char.ToUpper(chk);
106	if (!m_Chunks.ContainsKey(chk)) return;
107
108	WriteChunk(chk);
109
110	m_CurrentMemoryPressure -= m_ChunkMemoryPressure[chk];
111	m_ChunkMemoryPressure.Remove(chk);
112
113	m_Chunks.Remove(chk);
114	m_LastAccess.Remove(chk);
115	}
116
117	public void DumpAll() {
118	LastTuplesOut = 0;
119	lock (_dmpLock) {
120	if (!Directory.Exists(DirectoryName))
121	Directory.CreateDirectory(DirectoryName);
122
123	foreach (var item in m_Chunks.Keys.ToList()) {
124	WriteChunk(item);
125	}
126	}
127	}
128
129	protected void WriteChunk(char c) {
130	c = char.ToUpper(c);
131	if (!m_Chunks.ContainsKey(c)) return;
132
133	string fp = Path.Combine(DirectoryName, c + ".bin");
134
135	using (var str = File.Open(fp, FileMode.Create, FileAccess.Write))
136	using (var bw = new BinaryWriter(str, System.Text.Encoding.UTF8, false)) {
137	foreach (var kv in m_Chunks[c]) {
138	byte[] idBytes = System.Text.Encoding.UTF8.GetBytes(kv.Key);
139	bw.Write(idBytes.Length);
140	bw.Write(idBytes);
141
142	byte[] valueBytes = kv.Value ?? new byte[0];
143	bw.Write(valueBytes.Length);
144	bw.Write(valueBytes);
145	LastTuplesOut++;
146	}
147	bw.Write(0); // terminator
148	}
149	}
150
151	protected void LoadAllKeys() {
152	if (!Directory.Exists(DirectoryName)) return;
153
154	foreach (var chunk in Directory.GetFiles(DirectoryName, "*.bin")) {
155	var cn = Path.GetFileNameWithoutExtension(chunk)[0];
156	m_LastAccess[cn] = DateTime.MinValue;
157
158	using (var fs = File.OpenRead(chunk))
159	using (var br = new BinaryReader(fs, System.Text.Encoding.UTF8, true)) {
160	while (true) {
161	int idlen = br.ReadInt32();
162	if (idlen == 0) break;
163
164	string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idlen));
165	int ctlen = br.ReadInt32();
166	fs.Seek(ctlen, SeekOrigin.Current);
167
168	m_AllKeys.Add(id);
169	}
170	}
171	}
172	}
173
174	protected void PrepAddChunk(char chk) {
175	chk = char.ToUpper(chk);
176	if (!m_Chunks.ContainsKey(chk))
177	m_Chunks[chk] = new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
178
179	if (!m_ChunkMemoryPressure.ContainsKey(chk))
180	m_ChunkMemoryPressure[chk] = 0L;
181
182	m_LastAccess[chk] = DateTime.Now;
183	}
184
185	protected bool LoadChunk(char chk) {
186	chk = char.ToUpper(chk);
187	if (!Directory.Exists(DirectoryName)) return false;
188
189	string path = Path.Combine(DirectoryName, chk + ".bin");
190	if (!File.Exists(path)) return false;
191	if (m_Chunks.ContainsKey(chk)) return true;
192
193	PrepAddChunk(chk);
194
195	using (var str = File.OpenRead(path))
196	using (var br = new BinaryReader(str, System.Text.Encoding.UTF8, true)) {
197	while (true) {
198	int idl = br.ReadInt32();
199	if (idl == 0) break;
200
201	string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idl));
202	int ctlen = br.ReadInt32();
203	byte[] ctbytes = br.ReadBytes(ctlen);
204
205	m_Chunks[chk][id] = ctbytes;
206	m_AllKeys.Add(id);
207
208	long mp = id.Length + ctbytes.Length;
209	m_CurrentMemoryPressure += mp;
210	m_ChunkMemoryPressure[chk] += mp;
211	}
212	}
213
214	return true;
215	}
216	}
217
218	}