change to a more memory efficient map, implemented by several lists of

<key,offset,size>

weed-fs/src/pkg/storage/needle_map.go

@@ -6,20 +6,15 @@ import (
 	"pkg/util"
 )
 
-type NeedleValue struct {
-	Offset uint32 "Volume offset" //since aligned to 8 bytes, range is 4G*8=32G
-	Size   uint32 "Size of the data portion"
-}
-
 type NeedleMap struct {
 	indexFile *os.File
-	m         map[uint64]NeedleValue //mapping needle key(uint64) to NeedleValue
+	m         CompactMap
 	bytes     []byte
 }
 
 func NewNeedleMap(file *os.File) *NeedleMap {
 	nm := &NeedleMap{
-		m:         make(map[uint64]NeedleValue),
+		m:         NewCompactMap(),
 		bytes:     make([]byte, 16),
 		indexFile: file,
 	}
@@ -43,31 +38,31 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
 			key := util.BytesToUint64(bytes[i : i+8])
 			offset := util.BytesToUint32(bytes[i+8 : i+12])
 			size := util.BytesToUint32(bytes[i+12 : i+16])
-			if offset>0 {
-   			nm.m[key] = NeedleValue{util.Offset: offset, Size: size}
-   	  }else{
-        delete(nm.m, key)
-   	  }
-		}     
-		
+			if offset > 0 {
+				nm.m.Set(Key(key), offset, size)
+			} else {
+				nm.m.Delete(Key(key))
+			}
+		}
+
 		count, e = nm.indexFile.Read(bytes)
 	}
 	return nm
-}     
+}
 
 func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {
-	nm.m[key] = NeedleValue{Offset: offset, Size: size}
+	nm.m.Set(Key(key), offset, size)
 	util.Uint64toBytes(nm.bytes[0:8], key)
 	util.Uint32toBytes(nm.bytes[8:12], offset)
 	util.Uint32toBytes(nm.bytes[12:16], size)
 	return nm.indexFile.Write(nm.bytes)
 }
-func (nm *NeedleMap) Get(key uint64) (element NeedleValue, ok bool) {
-	element, ok = nm.m[key]
+func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
+	element, ok = nm.m.Get(Key(key))
 	return
 }
 func (nm *NeedleMap) Delete(key uint64) {
-	delete(nm.m, key)
+	nm.m.Delete(Key(key))
 	util.Uint64toBytes(nm.bytes[0:8], key)
 	util.Uint32toBytes(nm.bytes[8:12], 0)
 	util.Uint32toBytes(nm.bytes[12:16], 0)
@@ -76,6 +71,3 @@ func (nm *NeedleMap) Delete(key uint64) {
 func (nm *NeedleMap) Close() {
 	nm.indexFile.Close()
 }
-func (nm *NeedleMap) Length() int{
-  return len(nm.m)
-}