1 module maxmind.data; 2 import std.bitmanip : bigEndianToNative; 3 import std.conv : to; 4 import std.traits; 5 6 /** 7 * Generic container class that represents a metadata node for the Maxmind database format. 8 */ 9 public abstract class DataNode { 10 enum Type : ubyte { 11 Extended = 0, 12 Pointer = 1, 13 String = 2, 14 Double = 3, 15 Binary = 4, 16 Uint16 = 5, 17 Uint32 = 6, 18 Int32 = 8, 19 Uint64 = 9, 20 Uint128 = 10, 21 Map = 7, 22 Array = 11, 23 CacheContainer = 12, 24 EndMarker = 13, 25 Boolean = 14, 26 Float = 15 27 } 28 29 /** Holds the original type of the data */ 30 protected Type _type; 31 32 /** Constructs the node with the appropriate type tag */ 33 public this(Type type) { 34 this._type = type; 35 } 36 37 /** Readonly accessor for the node type */ 38 @property public Type type() const { 39 return this._type; 40 } 41 42 43 /** 44 * Creates a new node of the appropriate type using an existing reader helper as the data source 45 */ 46 public static DataNode create(ref Reader data) { 47 byte id = data.read(); 48 size_t payloadSize; 49 50 // Read object type 51 Type type = cast(Type)((id & 0b11100000) >> 5); 52 53 if(type == Type.Pointer) { 54 return DataNode.followPointer(id, data); 55 } 56 57 if(type == Type.Extended) { 58 type = cast(Type)(data.read() + 7); 59 } 60 61 // Read payload size 62 payloadSize = id & 0b00011111; 63 64 if(payloadSize >= 29) final switch(payloadSize) { 65 case 29: payloadSize = data.read!uint(1) + 29; break; 66 case 30: payloadSize = data.read!uint(2) + 285; break; 67 case 31: payloadSize = data.read!uint(3) + 65821; break; 68 } 69 70 // Construct the object node 71 switch(type) { 72 case Type.String: return new String (type, data, payloadSize); 73 case Type.Double: return new Number!double (type, data, payloadSize); 74 case Type.Binary: return new Binary (type, data, payloadSize); 75 case Type.Uint16: return new Number!ushort (type, data, payloadSize); 76 case Type.Uint32: return new Number!uint (type, data, payloadSize); 77 case Type.Map: return new Map (type, data, payloadSize); 78 case Type.Int32: return new Number!int (type, data, payloadSize); 79 case Type.Uint64: return new Number!ulong (type, data, payloadSize); 80 case Type.Uint128: return new Binary (type, data, payloadSize); 81 case Type.Array: return new Array (type, data, payloadSize); 82 case Type.CacheContainer: assert(false, "Unimplemented type: CacheContainer"); 83 case Type.EndMarker: assert(false, "Unimplemented type: EndMarker"); 84 case Type.Boolean: return new Boolean (type, data, payloadSize); 85 case Type.Float: return new Number!float (type, data, payloadSize); 86 default: 87 return new Binary(type, data, payloadSize); 88 } 89 } 90 91 /// ditto 92 public static DataNode create(Reader r) { 93 return DataNode.create(r); 94 } 95 96 /** 97 * Creates a new node using a raw data slice with an optional offset to the beginning of the data section. 98 * @see create(ref Reader data) 99 */ 100 public static DataNode create(ubyte[] data, size_t offset = 0) { 101 return DataNode.create(Reader(data, offset)); 102 } 103 104 /** 105 * Decodes and follows a pointer type 106 */ 107 public static DataNode followPointer(ubyte id, ref Reader data) { 108 uint extrabits = id & 0b00000111; 109 size_t jump; 110 111 final switch((id &0b00011000) >> 3) { 112 case 0: jump = data.read!uint(1, extrabits) + 0; break; 113 case 1: jump = data.read!uint(2, extrabits) + 2048; break; 114 case 2: jump = data.read!uint(3, extrabits) + 526336; break; 115 case 3: jump = data.read!uint(4); break; 116 } 117 118 return DataNode.create(data.newReader(jump)); 119 } 120 121 122 /** 123 * Generic binary data type 124 */ 125 public static class Binary : DataNode { 126 protected ubyte[] _data; 127 128 @property const(ubyte[]) data() const { 129 return this._data; 130 } 131 132 public this(Type type, ref Reader data, size_t payloadSize) { 133 super(type); 134 this._data = data.read(payloadSize); 135 } 136 } 137 138 139 /** 140 * Holds strings coming from the database 141 */ 142 public static class String : DataNode { 143 protected string _value; 144 145 public this(Type type, ref Reader data, size_t length) { 146 super(type); 147 this._value = cast(string) data.read(length); 148 } 149 150 @property public string value() { 151 return this._value; 152 } 153 } 154 155 156 /** 157 * Holds any kind of number from the database 158 */ 159 public static class Number(T) : DataNode { 160 protected T _value; 161 162 public this(Type type, ref Reader data, size_t actualSize) { 163 super(type); 164 this._value = data.read!T(actualSize); 165 } 166 167 @property public T value() { 168 return this._value; 169 } 170 } 171 172 173 /** 174 * Holds a boolean value from the database 175 */ 176 public static class Boolean : DataNode { 177 protected bool _value; 178 179 public this(Type type, ref Reader data, size_t value) { 180 super(type); 181 this._value = value > 0; 182 } 183 184 @property public bool value() { 185 return this._value; 186 } 187 } 188 189 190 /** 191 * Holds a Map structure from the database. 192 */ 193 public static class Map : DataNode { 194 DataNode[string] _map; 195 196 197 /** 198 * Constructs the map and loads all the key/value pairs 199 */ 200 public this(Type type, ref Reader data, size_t numValues) { 201 super(type); 202 203 while(numValues > 0) { 204 DataNode key = DataNode.create(data); 205 DataNode value = DataNode.create(data); 206 207 if(key.type != Type.String) { 208 throw new Exception("Invalid map key: expected string, got " ~ key.type.to!string ~ " instead."); 209 } 210 211 DataNode.String keyString = cast(DataNode.String) key; 212 this._map[keyString.value] = value; 213 numValues--; 214 } 215 } 216 217 218 /** Allow using foreach over the map's keys and values */ 219 public int opApply(int delegate(string, DataNode) dg) { 220 int result = 0; 221 222 foreach(key, node; this._map) { 223 result = dg(key, node); 224 if(result) return result; 225 } 226 227 return 0; 228 } 229 } 230 231 232 /** 233 * Holds an Array structure from the database. 234 */ 235 public static class Array : DataNode { 236 DataNode[] _values; 237 238 /** 239 * Constructs the Array as well as all its contained values 240 */ 241 public this(Type type, ref Reader data, size_t length) { 242 super(type); 243 244 while(length > 0) { 245 this._values ~= DataNode.create(data); 246 length--; 247 } 248 } 249 250 251 /** Returns the length of the array */ 252 @property public size_t length() { 253 return this._values.length; 254 } 255 256 257 /** Allow using foreach over the array of nodes */ 258 public int opApply(int delegate(size_t, DataNode) dg) { 259 int result = 0; 260 261 foreach(key, node; this._values) { 262 result = dg(key, node); 263 if(result) return result; 264 } 265 266 return 0; 267 } 268 } 269 270 271 /** 272 * Accesses the node values directly by automatically casting the node and getting its value 273 */ 274 public T get(T)() { 275 CastOutput conv(CastOutput)() { 276 CastOutput output = cast(CastOutput)(this); 277 278 if(output is null) { 279 throw new Exception("Invalid conversion from " ~ this.type.to!string ~ " to " ~ T.stringof); 280 } 281 282 return output; 283 } 284 285 static if(is(T == string)) return conv!String.value; 286 else static if(is(T == double)) return conv!(Number!double).value; 287 else static if(is(T == ubyte[])) return conv!Binary.data; 288 else static if(is(T == ushort)) return conv!(Number!ushort).value; 289 else static if(is(T == uint)) return conv!(Number!uint).value; 290 else static if(is(T == int)) return conv!(Number!int).value; 291 else static if(is(T == ulong)) return conv!(Number!ulong).value; 292 else static if(is(T == bool)) return conv!Boolean.value; 293 else static if(is(T == float)) return conv!(Number!float).value; 294 else { 295 static assert(false, "Unknown output conversion to " ~ T.stringof); 296 } 297 } 298 299 300 /** 301 * Attemps to convert the current node to a Map node 302 */ 303 public Map asMap() { 304 Map m = cast(Map) this; 305 306 if(m is null) { 307 throw new Exception("Using a node of type " ~ this.type.to!string ~ " as a map."); 308 } 309 310 return m; 311 } 312 313 /** 314 * Provide a way to export an entire map as an associative array of the specified type 315 */ 316 public T[string] getMap(T)() { 317 Map m = this.asMap(); 318 T[string] map; 319 320 foreach(key, node; m._map) { 321 map[key] = node.get!T; 322 } 323 324 return map; 325 } 326 327 328 /** 329 * Attempts to convert the current node to an Array node 330 */ 331 public Array asArray() { 332 Array a = cast(Array) this; 333 334 if(a is null) { 335 throw new Exception("Using a node of type " ~ this.type.to!string ~ " as an array"); 336 } 337 return a; 338 } 339 340 /** 341 * Provide a way to export the entire array as a specific type 342 */ 343 public T[] getArray(T)() { 344 Array a = this.asArray(); 345 T[] arr = new T[a._values.length]; 346 347 for(size_t i = 0; i < arr.length; i++) { 348 arr[i] = a._values[i].get!T; 349 } 350 351 return arr; 352 } 353 354 355 /** 356 * Allow accessing Map values using dot notation 357 * Note: I put the method here to avoid having to cast objets all over the place 358 */ 359 public DataNode opDispatch(string key)() { 360 if(key !in this.asMap()._map) { 361 return null; 362 } 363 364 return this.asMap()._map[key]; 365 } 366 367 368 /** 369 * Allow accessing Map values using the [] operator using string keys 370 * Note: I put the method here to avoid having to cast objets all over the place 371 */ 372 public DataNode opIndex(string key) { 373 if(key !in this.asMap()._map) { 374 return null; 375 } 376 377 return this.asMap()._map[key]; 378 } 379 380 381 /** 382 * Allow accessing Array values using the [] operator using ulong keys 383 * Note: I put the method here to avoid having to cast objets all over the place 384 */ 385 public DataNode opIndex(size_t key) { 386 return this.asArray()._values[key]; 387 } 388 } 389 390 391 /** 392 * Helper container that manages changing position in the mmapped database file. It automatically handles 393 * moving the internal pointer forward when data is read to ensure proper alignment of data. 394 */ 395 package struct Reader { 396 /** The raw slice of the whole database */ 397 ubyte[] data; 398 399 /** Pointer to the current offset in the database file */ 400 size_t current; 401 402 403 /** 404 * Constructs a new reader from a slice and a starting offset 405 */ 406 this(ubyte[] data, size_t startOffset = 0) { 407 this.data = data; 408 this.current = startOffset; 409 } 410 411 412 /** 413 * Creates a new reader at a different offset: used by the pointer and cache types to go fetch data 414 * outside of their own location. 415 */ 416 Reader newReader(size_t offset) { 417 return Reader(this.data, offset); 418 } 419 420 421 /** 422 * Reads $length bytes from the database 423 */ 424 ubyte[] read(size_t length) { 425 size_t start = this.current; 426 this.current += length; 427 return this.data[start..this.current]; 428 } 429 430 431 /** 432 * Reads $length bytes from the database and convert it to a native type of the appropriate endianness 433 */ 434 T read(T)(size_t length) { 435 ubyte[T.sizeof] buffer = 0; 436 buffer[$-cast(size_t)length..$] = this.read(length); 437 return bigEndianToNative!T(buffer); 438 } 439 440 441 /** 442 * Reads $length byte into an integer type with the appropriate endianness and append extra most significant bits 443 */ 444 T read(T)(size_t length, T extrabits) if(isIntegral!T) { 445 return this.read!T(length) + (extrabits << (length*8)); 446 } 447 448 449 /** 450 * Reads a single byte from the database 451 */ 452 ubyte read() { 453 return this.data[this.current++]; 454 } 455 }