trunk/TELTest/BasicKeyValue.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace TELTest {
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;

    internal class BasicKeyValue {
        public BasicKeyValue(string directoryName) {
            DirectoryName = directoryName;
            LoadAllKeys();
        }

        public long MaximumMemoryPressure { get; set; } = 2L * 1024 * 1024 * 1024; // 2 GB
        public long CurrentMemoryPressure => m_CurrentMemoryPressure;
        public string DirectoryName { get; set; } = "Database";
        public int LastTuplesOut { get; protected set; }
        private long m_CurrentMemoryPressure = 0;
        private readonly object _dmpLock = new object();

        private readonly Dictionary<char, DateTime> m_LastAccess = new Dictionary<char, DateTime>();
        private readonly HashSet<string> m_AllKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        private readonly Dictionary<char, long> m_ChunkMemoryPressure = new Dictionary<char, long>();
        private readonly Dictionary<char, Dictionary<string, byte[]>> m_Chunks = new Dictionary<char, Dictionary<string, byte[]>>();

        public bool ContainsKey(string key) => m_AllKeys.Contains(key);

        public byte[] this[string key] {
            get {
                char q = char.ToUpper(key[0]);

                if (!m_AllKeys.Contains(key))
                    return null;

                if (!m_Chunks.ContainsKey(q) || !m_Chunks[q].ContainsKey(key)) {
                    LoadAndFreeIfNeeded(q);
                }

                if (m_Chunks.ContainsKey(q) && m_Chunks[q].ContainsKey(key)) {
                    m_LastAccess[q] = DateTime.Now;
                    return m_Chunks[q][key];
                }

                return null;
            }
            set => AddOrSet(key, value);
        }

        public void Remove(string key) {
            char c = char.ToUpper(key[0]);
            if (!m_AllKeys.Remove(key)) return;

            if (m_Chunks.ContainsKey(c)) {
                var oldValue = m_Chunks[c][key];
                m_Chunks[c].Remove(key);
                m_ChunkMemoryPressure[c] -= oldValue.Length;
                m_CurrentMemoryPressure -= oldValue.Length;
            } else {
                LoadAndFreeIfNeeded(c);
                if (m_Chunks[c].ContainsKey(key)) {
                    var loadedValue = m_Chunks[c][key];
                    if (m_Chunks[c].Remove(key)) {
                        m_ChunkMemoryPressure[c] -= loadedValue.Length;
                        m_CurrentMemoryPressure -= loadedValue.Length;
                    }
                }
            }
        }

        public void SavePendingChanges() => DumpAll();

        protected void AddOrSet(string key, byte[] value) {
            char q = char.ToUpper(key[0]);

            PrepAddChunk(q);

            if (m_Chunks[q].TryGetValue(key, out var oldValue)) {
                m_Chunks[q][key] = value;
                long delta = value.Length - oldValue.Length;
                m_ChunkMemoryPressure[q] += delta;
                m_CurrentMemoryPressure += delta;
            } else {
                m_AllKeys.Add(key);
                m_Chunks[q].Add(key, value);
                m_ChunkMemoryPressure[q] += value.Length;
                m_CurrentMemoryPressure += value.Length;
            }
        }

        protected void LoadAndFreeIfNeeded(char at) {
            while (m_CurrentMemoryPressure > MaximumMemoryPressure && m_LastAccess.Any()) {
                var oldest = m_LastAccess.OrderBy(x => x.Value).First();
                UnloadChunk(oldest.Key);
            }
            LoadChunk(at);
        }

        protected void UnloadChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!m_Chunks.ContainsKey(chk)) return;

            WriteChunk(chk);

            m_CurrentMemoryPressure -= m_ChunkMemoryPressure[chk];
            m_ChunkMemoryPressure.Remove(chk);

            m_Chunks.Remove(chk);
            m_LastAccess.Remove(chk);
        }

        public void DumpAll() {
            LastTuplesOut = 0;
            lock (_dmpLock) {
                if (!Directory.Exists(DirectoryName))
                    Directory.CreateDirectory(DirectoryName);

                foreach (var item in m_Chunks.Keys.ToList()) {
                    WriteChunk(item);
                }
            }
        }

        protected void WriteChunk(char c) {
            c = char.ToUpper(c);
            if (!m_Chunks.ContainsKey(c)) return;

            string fp = Path.Combine(DirectoryName, c + ".bin");

            using (var str = File.Open(fp, FileMode.Create, FileAccess.Write))
            using (var bw = new BinaryWriter(str, System.Text.Encoding.UTF8, false)) {
                foreach (var kv in m_Chunks[c]) {
                    byte[] idBytes = System.Text.Encoding.UTF8.GetBytes(kv.Key);
                    bw.Write(idBytes.Length);
                    bw.Write(idBytes);

                    byte[] valueBytes = kv.Value ?? new byte[0];
                    bw.Write(valueBytes.Length);
                    bw.Write(valueBytes);
                    LastTuplesOut++;
                }
                bw.Write(0); // terminator
            }
        }

        protected void LoadAllKeys() {
            if (!Directory.Exists(DirectoryName)) return;

            foreach (var chunk in Directory.GetFiles(DirectoryName, "*.bin")) {
                var cn = Path.GetFileNameWithoutExtension(chunk)[0];
                m_LastAccess[cn] = DateTime.MinValue;

                using (var fs = File.OpenRead(chunk))
                using (var br = new BinaryReader(fs, System.Text.Encoding.UTF8, true)) {
                    while (true) {
                        int idlen = br.ReadInt32();
                        if (idlen == 0) break;

                        string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idlen));
                        int ctlen = br.ReadInt32();
                        fs.Seek(ctlen, SeekOrigin.Current);

                        m_AllKeys.Add(id);
                    }
                }
            }
        }

        protected void PrepAddChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!m_Chunks.ContainsKey(chk))
                m_Chunks[chk] = new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);

            if (!m_ChunkMemoryPressure.ContainsKey(chk))
                m_ChunkMemoryPressure[chk] = 0L;

            m_LastAccess[chk] = DateTime.Now;
        }

        protected bool LoadChunk(char chk) {
            chk = char.ToUpper(chk);
            if (!Directory.Exists(DirectoryName)) return false;

            string path = Path.Combine(DirectoryName, chk + ".bin");
            if (!File.Exists(path)) return false;
            if (m_Chunks.ContainsKey(chk)) return true;

            PrepAddChunk(chk);

            using (var str = File.OpenRead(path))
            using (var br = new BinaryReader(str, System.Text.Encoding.UTF8, true)) {
                while (true) {
                    int idl = br.ReadInt32();
                    if (idl == 0) break;

                    string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idl));
                    int ctlen = br.ReadInt32();
                    byte[] ctbytes = br.ReadBytes(ctlen);

                    m_Chunks[chk][id] = ctbytes;
                    m_AllKeys.Add(id);

                    long mp = id.Length + ctbytes.Length;
                    m_CurrentMemoryPressure += mp;
                    m_ChunkMemoryPressure[chk] += mp;
                }
            }

            return true;
        }
    }

}
Revision:	2
Committed:	Wed Feb 11 13:07:01 2026 UTC (8 weeks, 3 days ago) by figdor32
File size:	7999 byte(s)
Log Message:	Initial check-in
#	User	Rev	Content
1	figdor32	2	using System;
2			using System.Collections.Generic;
3			using System.IO;
4			using System.Linq;
5			using System.Text;
6			using System.Threading.Tasks;
7
8			namespace TELTest {
9			using System;
10			using System.Collections.Generic;
11			using System.IO;
12			using System.Linq;
13
14			internal class BasicKeyValue {
15			public BasicKeyValue(string directoryName) {
16			DirectoryName = directoryName;
17			LoadAllKeys();
18			}
19
20			public long MaximumMemoryPressure { get; set; } = 2L * 1024 * 1024 * 1024; // 2 GB
21			public long CurrentMemoryPressure => m_CurrentMemoryPressure;
22			public string DirectoryName { get; set; } = "Database";
23			public int LastTuplesOut { get; protected set; }
24			private long m_CurrentMemoryPressure = 0;
25			private readonly object _dmpLock = new object();
26
27			private readonly Dictionary<char, DateTime> m_LastAccess = new Dictionary<char, DateTime>();
28			private readonly HashSet<string> m_AllKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
29			private readonly Dictionary<char, long> m_ChunkMemoryPressure = new Dictionary<char, long>();
30			private readonly Dictionary<char, Dictionary<string, byte[]>> m_Chunks = new Dictionary<char, Dictionary<string, byte[]>>();
31
32			public bool ContainsKey(string key) => m_AllKeys.Contains(key);
33
34			public byte[] this[string key] {
35			get {
36			char q = char.ToUpper(key[0]);
37
38			if (!m_AllKeys.Contains(key))
39			return null;
40
41			if (!m_Chunks.ContainsKey(q) \|\| !m_Chunks[q].ContainsKey(key)) {
42			LoadAndFreeIfNeeded(q);
43			}
44
45			if (m_Chunks.ContainsKey(q) && m_Chunks[q].ContainsKey(key)) {
46			m_LastAccess[q] = DateTime.Now;
47			return m_Chunks[q][key];
48			}
49
50			return null;
51			}
52			set => AddOrSet(key, value);
53			}
54
55			public void Remove(string key) {
56			char c = char.ToUpper(key[0]);
57			if (!m_AllKeys.Remove(key)) return;
58
59			if (m_Chunks.ContainsKey(c)) {
60			var oldValue = m_Chunks[c][key];
61			m_Chunks[c].Remove(key);
62			m_ChunkMemoryPressure[c] -= oldValue.Length;
63			m_CurrentMemoryPressure -= oldValue.Length;
64			} else {
65			LoadAndFreeIfNeeded(c);
66			if (m_Chunks[c].ContainsKey(key)) {
67			var loadedValue = m_Chunks[c][key];
68			if (m_Chunks[c].Remove(key)) {
69			m_ChunkMemoryPressure[c] -= loadedValue.Length;
70			m_CurrentMemoryPressure -= loadedValue.Length;
71			}
72			}
73			}
74			}
75
76			public void SavePendingChanges() => DumpAll();
77
78			protected void AddOrSet(string key, byte[] value) {
79			char q = char.ToUpper(key[0]);
80
81			PrepAddChunk(q);
82
83			if (m_Chunks[q].TryGetValue(key, out var oldValue)) {
84			m_Chunks[q][key] = value;
85			long delta = value.Length - oldValue.Length;
86			m_ChunkMemoryPressure[q] += delta;
87			m_CurrentMemoryPressure += delta;
88			} else {
89			m_AllKeys.Add(key);
90			m_Chunks[q].Add(key, value);
91			m_ChunkMemoryPressure[q] += value.Length;
92			m_CurrentMemoryPressure += value.Length;
93			}
94			}
95
96			protected void LoadAndFreeIfNeeded(char at) {
97			while (m_CurrentMemoryPressure > MaximumMemoryPressure && m_LastAccess.Any()) {
98			var oldest = m_LastAccess.OrderBy(x => x.Value).First();
99			UnloadChunk(oldest.Key);
100			}
101			LoadChunk(at);
102			}
103
104			protected void UnloadChunk(char chk) {
105			chk = char.ToUpper(chk);
106			if (!m_Chunks.ContainsKey(chk)) return;
107
108			WriteChunk(chk);
109
110			m_CurrentMemoryPressure -= m_ChunkMemoryPressure[chk];
111			m_ChunkMemoryPressure.Remove(chk);
112
113			m_Chunks.Remove(chk);
114			m_LastAccess.Remove(chk);
115			}
116
117			public void DumpAll() {
118			LastTuplesOut = 0;
119			lock (_dmpLock) {
120			if (!Directory.Exists(DirectoryName))
121			Directory.CreateDirectory(DirectoryName);
122
123			foreach (var item in m_Chunks.Keys.ToList()) {
124			WriteChunk(item);
125			}
126			}
127			}
128
129			protected void WriteChunk(char c) {
130			c = char.ToUpper(c);
131			if (!m_Chunks.ContainsKey(c)) return;
132
133			string fp = Path.Combine(DirectoryName, c + ".bin");
134
135			using (var str = File.Open(fp, FileMode.Create, FileAccess.Write))
136			using (var bw = new BinaryWriter(str, System.Text.Encoding.UTF8, false)) {
137			foreach (var kv in m_Chunks[c]) {
138			byte[] idBytes = System.Text.Encoding.UTF8.GetBytes(kv.Key);
139			bw.Write(idBytes.Length);
140			bw.Write(idBytes);
141
142			byte[] valueBytes = kv.Value ?? new byte[0];
143			bw.Write(valueBytes.Length);
144			bw.Write(valueBytes);
145			LastTuplesOut++;
146			}
147			bw.Write(0); // terminator
148			}
149			}
150
151			protected void LoadAllKeys() {
152			if (!Directory.Exists(DirectoryName)) return;
153
154			foreach (var chunk in Directory.GetFiles(DirectoryName, "*.bin")) {
155			var cn = Path.GetFileNameWithoutExtension(chunk)[0];
156			m_LastAccess[cn] = DateTime.MinValue;
157
158			using (var fs = File.OpenRead(chunk))
159			using (var br = new BinaryReader(fs, System.Text.Encoding.UTF8, true)) {
160			while (true) {
161			int idlen = br.ReadInt32();
162			if (idlen == 0) break;
163
164			string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idlen));
165			int ctlen = br.ReadInt32();
166			fs.Seek(ctlen, SeekOrigin.Current);
167
168			m_AllKeys.Add(id);
169			}
170			}
171			}
172			}
173
174			protected void PrepAddChunk(char chk) {
175			chk = char.ToUpper(chk);
176			if (!m_Chunks.ContainsKey(chk))
177			m_Chunks[chk] = new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
178
179			if (!m_ChunkMemoryPressure.ContainsKey(chk))
180			m_ChunkMemoryPressure[chk] = 0L;
181
182			m_LastAccess[chk] = DateTime.Now;
183			}
184
185			protected bool LoadChunk(char chk) {
186			chk = char.ToUpper(chk);
187			if (!Directory.Exists(DirectoryName)) return false;
188
189			string path = Path.Combine(DirectoryName, chk + ".bin");
190			if (!File.Exists(path)) return false;
191			if (m_Chunks.ContainsKey(chk)) return true;
192
193			PrepAddChunk(chk);
194
195			using (var str = File.OpenRead(path))
196			using (var br = new BinaryReader(str, System.Text.Encoding.UTF8, true)) {
197			while (true) {
198			int idl = br.ReadInt32();
199			if (idl == 0) break;
200
201			string id = System.Text.Encoding.UTF8.GetString(br.ReadBytes(idl));
202			int ctlen = br.ReadInt32();
203			byte[] ctbytes = br.ReadBytes(ctlen);
204
205			m_Chunks[chk][id] = ctbytes;
206			m_AllKeys.Add(id);
207
208			long mp = id.Length + ctbytes.Length;
209			m_CurrentMemoryPressure += mp;
210			m_ChunkMemoryPressure[chk] += mp;
211			}
212			}
213
214			return true;
215			}
216			}
217
218			}