badgerdb messageservers storage

.gitignore

@@ -15,3 +15,4 @@ client/test.cfg
 .VSCodeCouter/
 meowlib-sources.jar
 meowlib.aar
+client/test.db

README.md

@@ -22,7 +22,7 @@ run the shell scripts
 # Design notes
 Config is written as a json file
 Identity is stored as an encrypted json file
-Prefered servers (messaging and my contact's messaging) are stored in an encrypted badger db with server url as key
-Received servers are stored in a sqlite db for extensive searches, with storage limits
+Message servers (messaging and my contact's messaging) are stored in an encrypted badger db with server url as key
+Received servers are stored in a sqlite db for selective searches, with storage limits
 Messages are stored in several badger? or sqlite? db per user with send/receive time as key
 

client/identity.go

@@ -21,11 +21,10 @@ type Identity struct {
 	RootKp               meowlib.KeyPair `json:"id_kp,omitempty"`
 	Status               string          `json:"status,omitempty"`
 	Peers                PeerList        `json:"peers,omitempty"`
-	Devices              PeerList        `json:"devices,omitempty"`
 	HiddenPeers          [][]byte        `json:"hidden_peers,omitempty"`
 	Device               meowlib.KeyPair `json:"device,omitempty"`
 	KnownServers         ServerList      `json:"known_servers,omitempty"`
-	MessageServers       ServerList      `json:"message_servers,omitempty"`
+	MessageServers       ServerStorage   `json:"message_servers,omitempty"`
 	DefaultDbPassword    string          `json:"default_db_password,omitempty"`
 	DbPasswordStore      bool            `json:"db_password_store,omitempty"`
 	OwnedDevices         PeerList        `json:"owned_devices,omitempty"`
@@ -40,19 +39,20 @@ func CreateIdentity(nickname string) *Identity {
 	id.Nickname = nickname
 	id.RootKp = meowlib.NewKeyPair()
 	GetConfig().me = &id
+	id.MessageServers = ServerStorage{DbFile: uuid.NewString()}
 	id.generateRandomHiddenStuff()
 	return &id
 }
 
-// Creates an invitation for a peer, returns the peer containing
-func (id *Identity) InvitePeer(MyName string, ContactName string, MessageServerIdxs []int, InvitationMessage string) (*Peer, error) {
+// Creates an invitation for a peer, returns the newly created peer including infos to provide a ContactCard
+func (id *Identity) InvitePeer(MyName string, ContactName string, MessageServerUids []string, InvitationMessage string) (*Peer, error) {
 	var peer Peer
 	peer.MyIdentity = meowlib.NewKeyPair()
 	peer.MyEncryptionKp = meowlib.NewKeyPair()
 	peer.MyLookupKp = meowlib.NewKeyPair()
 	peer.Name = ContactName
 	peer.InvitationId = uuid.New().String() // todo as param to identify then update url
-	if id.MessageServers.Servers == nil {
+	/*	if id.MessageServers.Servers == nil {
 			return nil, errors.New("no message servers defined in your identity")
 		}
 		for _, i := range MessageServerIdxs {
@@ -63,7 +63,12 @@ func (id *Identity) InvitePeer(MyName string, ContactName string, MessageServerI
 		for _, i := range MessageServerIdxs {
 			srv := id.MessageServers.Servers[i].GetServerCard()
 			peer.MyContact.PullServers = append(peer.MyContact.PullServers, srv)
+		}*/
+	pullServers, err := id.MessageServers.LoadServerCardsFromUids(MessageServerUids)
+	if err != nil {
+		return nil, err
 	}
+	peer.MyContact.PullServers = pullServers
 	peer.MyContact.Name = MyName
 	peer.MyContact.ContactPublicKey = peer.MyIdentity.Public
 	peer.MyContact.EncryptionPublicKey = peer.MyEncryptionKp.Public
@@ -75,6 +80,7 @@ func (id *Identity) InvitePeer(MyName string, ContactName string, MessageServerI
 	return &peer, nil
 }
 
+// Checks if the received contact card is an answer to an invitation, returns true if it is, and the proposed and received nicknames
 func (id *Identity) CheckInvitation(ReceivedContact *meowlib.ContactCard) (isAnswer bool, proposedNick string, receivedNick string) {
 	for _, p := range id.Peers {
 		if p.InvitationId == ReceivedContact.InvitationId {
@@ -84,7 +90,8 @@ func (id *Identity) CheckInvitation(ReceivedContact *meowlib.ContactCard) (isAns
 	return false, "", ReceivedContact.Name
 }
 
-func (id *Identity) AnswerInvitation(MyName string, ContactName string, MessageServerIdxs []int, ReceivedContact *meowlib.ContactCard) *Peer {
+// Answers an invitation, returns the newly created peer including infos to provide a ContactCard
+func (id *Identity) AnswerInvitation(MyName string, ContactName string, MessageServerIdxs []string, ReceivedContact *meowlib.ContactCard) *Peer {
 	var peer Peer
 	//var myContactCard meowlib.ContactCard
 	peer.MyIdentity = meowlib.NewKeyPair()
@@ -96,9 +103,13 @@ func (id *Identity) AnswerInvitation(MyName string, ContactName string, MessageS
 		peer.Name = ReceivedContact.Name
 	}
 	peer.Contact = *ReceivedContact
-	for _, i := range MessageServerIdxs {
+	/*	for _, i := range MessageServerIdxs {
 		srv := id.MessageServers.Servers[i].GetServerCard()
 		peer.MyContact.PullServers = append(peer.MyContact.PullServers, srv)
+	}*/
+	srvCards, err := GetConfig().GetIdentity().MessageServers.LoadServerCardsFromUids(MessageServerIdxs)
+	if err != nil {
+		peer.MyContact.PullServers = srvCards
 	}
 	peer.MyContact.Name = MyName
 	peer.MyContact.ContactPublicKey = peer.MyIdentity.Public
@@ -111,6 +122,7 @@ func (id *Identity) AnswerInvitation(MyName string, ContactName string, MessageS
 	return &peer
 }
 
+// Finalizes an invitation, returns nil if successful
 func (id *Identity) FinalizeInvitation(ReceivedContact *meowlib.ContactCard) error {
 	for i, p := range id.Peers {
 		if p.InvitationId == ReceivedContact.InvitationId {
@@ -121,6 +133,7 @@ func (id *Identity) FinalizeInvitation(ReceivedContact *meowlib.ContactCard) err
 	return errors.New("no matching contact found for invitationId " + ReceivedContact.InvitationId)
 }
 
+// LoadIdentity loads an identity from an encrypted file
 func LoadIdentity(filename string, password string) (*Identity, error) {
 	var id Identity
 	GetConfig().memoryPassword = password
@@ -134,6 +147,9 @@ func LoadIdentity(filename string, password string) (*Identity, error) {
 		return nil, err
 	}
 	err = json.Unmarshal([]byte(pass), &id)
+	if err != nil {
+		return nil, err
+	}
 	GetConfig().me = &id
 	return &id, err
 }
@@ -215,7 +231,7 @@ type BackgroundJob struct {
 }
 
 type RequestsJob struct {
-	Server     Server            `json:"server,omitempty"`
+	Server     *Server           `json:"server,omitempty"`
 	LookupKeys []meowlib.KeyPair `json:"lookup_keys,omitempty"`
 }
 
@@ -223,7 +239,9 @@ func (id *Identity) GetRequestJobs() []*RequestsJob {
 	var list []*RequestsJob
 	srvs := map[string]*RequestsJob{}
 	// build a server map
-	for _, server := range id.MessageServers.Servers {
+	servers, err := id.MessageServers.LoadAllServers()
+	if err != nil {
+		for _, server := range servers {
 			var rj RequestsJob
 			rj.Server = server
 			srvs[server.GetServerCard().GetUid()] = &rj
@@ -231,13 +249,13 @@ func (id *Identity) GetRequestJobs() []*RequestsJob {
 		// add ids to the map
 		for _, peer := range id.Peers {
 			for _, server := range peer.MyPullServers {
-			srvs[server.GetUid()].LookupKeys = append(srvs[server.GetUid()].LookupKeys, peer.MyLookupKp)
+				srvs[server].LookupKeys = append(srvs[server].LookupKeys, peer.MyLookupKp)
 			}
 		}
 		// add hidden peers
 		for _, peer := range id.unlockedHiddenPeers {
 			for _, server := range peer.MyPullServers {
-			srvs[server.GetUid()].LookupKeys = append(srvs[server.GetUid()].LookupKeys, peer.MyLookupKp)
+				srvs[server].LookupKeys = append(srvs[server].LookupKeys, peer.MyLookupKp)
 			}
 		}
 		// todo add garbage
@@ -248,6 +266,7 @@ func (id *Identity) GetRequestJobs() []*RequestsJob {
 		for _, value := range srvs {
 			list = append(list, value)
 		}
+	}
 	return list
 }
 

client/peer.go

@@ -24,7 +24,7 @@ type Peer struct {
 	MyIdentity     meowlib.KeyPair     `json:"my_identity,omitempty"`
 	MyEncryptionKp meowlib.KeyPair     `json:"my_encryption_kp,omitempty"`
 	MyLookupKp     meowlib.KeyPair     `json:"my_lookup_kp,omitempty"`
-	MyPullServers  []*meowlib.ServerCard `json:"my_pull_servers,omitempty"`
+	MyPullServers  []string            `json:"my_pull_servers,omitempty"`
 	MyContact      meowlib.ContactCard `json:"my_contact,omitempty"` // todo : remove
 	// Peer keys and infos
 	Contact            meowlib.ContactCard `json:"contact,omitempty"` // todo : remove
@@ -59,7 +59,7 @@ func (p *Peer) GetMyContact() *meowlib.ContactCard {
 	c.ContactPublicKey = p.MyIdentity.Public
 	c.LookupPublicKey = p.MyLookupKp.Public
 	c.EncryptionPublicKey = p.MyEncryptionKp.Public
-	c.PullServers = p.MyPullServers
+	//	c.PullServers = p.MyPullServers
 	c.InvitationId = p.InvitationId
 	c.InvitationMessage = p.InvitationMessage
 	c.Name = p.MyName
@@ -67,12 +67,14 @@ func (p *Peer) GetMyContact() *meowlib.ContactCard {
 }
 
 func (p *Peer) GetContact() *meowlib.ContactCard {
-
 	var c meowlib.ContactCard
 	c.ContactPublicKey = p.ContactPublicKey
 	c.LookupPublicKey = p.ContactLookupKey
 	c.EncryptionPublicKey = p.ContactEncryption
-	//	c.PullServers = meowlib.LoadServerCardsFromUids(p.ContactPullServers)
+	srvCards, err := GetConfig().GetIdentity().MessageServers.LoadServerCardsFromUids(p.ContactPullServers)
+	if err != nil {
+		c.PullServers = srvCards
+	}
 	c.InvitationId = p.InvitationId
 	c.InvitationMessage = p.InvitationMessage
 	c.Name = p.Name

client/server.go

@@ -53,6 +53,13 @@ func (ints *Server) GetServerCard() *meowlib.ServerCard {
 	return &sc
 }
 
+func (sc *Server) GetUid() string {
+	if len(sc.Login) > 0 || len(sc.Password) > 0 {
+		return sc.Login + ":" + sc.Password + "@" + sc.Url
+	}
+	return sc.Url
+}
+
 // Create a server from a server card
 func CreateServerFromServerCard(server *meowlib.ServerCard) *Server {
 	var is Server

client/serverstorage.go

@@ -0,0 +1,200 @@
+package client
+
+//
+// Storage
+//
+import (
+	"crypto/sha256"
+	"encoding/json"
+
+	"forge.redroom.link/yves/meowlib"
+	"github.com/dgraph-io/badger"
+)
+
+type ServerStorage struct {
+	DbFile string `json:"db_file,omitempty"`
+	db     *badger.DB
+}
+
+// Open a badger database from struct ServerStorage
+func (ss *ServerStorage) open() error {
+	opts := badger.DefaultOptions(ss.DbFile)
+	opts.Logger = nil
+	var err error
+	ss.db, err = badger.Open(opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Store function StoreServer stores a server in a badger database with Server.GetUid() as key
+func (ss *ServerStorage) StoreServer(sc *Server) error {
+	err := ss.open()
+	if err != nil {
+		return err
+	}
+	defer ss.close()
+	// first marshal the Server to bytes with protobuf
+	jsonsrv, err := json.Marshal(sc)
+	if err != nil {
+		return err
+	}
+	data, err := meowlib.SymEncrypt(GetConfig().memoryPassword, jsonsrv)
+	if err != nil {
+		return err
+	}
+	shakey := sha256.Sum256([]byte(sc.GetServerCard().GetUid()))
+	key := shakey[:]
+	// then store it in the database
+	return ss.db.Update(func(txn *badger.Txn) error {
+		return txn.Set(key, data)
+	})
+
+}
+
+// LoadServer function loads a Server from a badger database with Server.GetUid() as key
+func (ss *ServerStorage) LoadServer(uid string) (*Server, error) {
+	var sc Server
+	err := ss.open()
+	if err != nil {
+		return nil, err
+	}
+	defer ss.close()
+	shakey := sha256.Sum256([]byte(uid))
+	key := shakey[:]
+	err = ss.db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(key)
+		if err != nil {
+			return err
+		}
+		return item.Value(func(val []byte) error {
+			jsonsrv, err := meowlib.SymDecrypt(GetConfig().memoryPassword, val)
+			if err != nil {
+				return err
+			}
+			return json.Unmarshal(jsonsrv, &sc)
+		})
+	})
+	return &sc, err
+}
+
+// DeleteServer function deletes a Server from a badger database with Server.GetUid() as key
+func (ss *ServerStorage) DeleteServer(uid string) error {
+	err := ss.open()
+	if err != nil {
+		return err
+	}
+	defer ss.close()
+	shakey := sha256.Sum256([]byte(uid))
+	key := shakey[:]
+	return ss.db.Update(func(txn *badger.Txn) error {
+		return txn.Delete(key)
+	})
+}
+
+// LoadAllServers function loads all Servers from a badger database
+func (ss *ServerStorage) LoadAllServers() ([]*Server, error) {
+	var scs []*Server
+	err := ss.open()
+	if err != nil {
+		return nil, err
+	}
+	defer ss.close()
+	err = ss.db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchSize = 10
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		for it.Rewind(); it.Valid(); it.Next() {
+			item := it.Item()
+			var sc Server
+			err := item.Value(func(val []byte) error {
+				jsonsrv, err := meowlib.SymDecrypt(GetConfig().memoryPassword, val)
+				if err != nil {
+					return err
+				}
+				return json.Unmarshal(jsonsrv, &sc)
+			})
+			if err != nil {
+				return err
+			}
+			scs = append(scs, &sc)
+		}
+		return nil
+	})
+	return scs, err
+}
+
+// LoadServersFromUids function loads Servers with id in []Uid parameter from a badger database
+func (ss *ServerStorage) LoadServersFromUids(uids []string) ([]*Server, error) {
+	var scs []*Server
+	err := ss.open()
+	if err != nil {
+		return nil, err
+	}
+	defer ss.close()
+	err = ss.db.View(func(txn *badger.Txn) error {
+		for _, uid := range uids {
+			shakey := sha256.Sum256([]byte(uid))
+			key := shakey[:]
+			item, err := txn.Get(key)
+			if err != nil {
+				return err
+			}
+			var sc Server
+			err = item.Value(func(val []byte) error {
+				jsonsrv, err := meowlib.SymDecrypt(GetConfig().memoryPassword, val)
+				if err != nil {
+					return err
+				}
+				return json.Unmarshal(jsonsrv, &sc)
+			})
+			if err != nil {
+				return err
+			}
+			scs = append(scs, &sc)
+		}
+		return nil
+	})
+	return scs, err
+}
+
+// LoadServersFromUids function loads Servers with id in []Uid parameter from a badger database
+func (ss *ServerStorage) LoadServerCardsFromUids(uids []string) ([]*meowlib.ServerCard, error) {
+	var scs []*meowlib.ServerCard
+	err := ss.open()
+	if err != nil {
+		return nil, err
+	}
+	defer ss.close()
+	err = ss.db.View(func(txn *badger.Txn) error {
+		for _, uid := range uids {
+			shakey := sha256.Sum256([]byte(uid))
+			key := shakey[:]
+			item, err := txn.Get(key)
+			if err != nil {
+				return err
+			}
+			var sc Server
+			err = item.Value(func(val []byte) error {
+				jsonsrv, err := meowlib.SymDecrypt(GetConfig().memoryPassword, val)
+				if err != nil {
+					return err
+				}
+				return json.Unmarshal(jsonsrv, &sc)
+			})
+			if err != nil {
+				return err
+			}
+			scs = append(scs, sc.GetServerCard())
+		}
+		return nil
+	})
+	return scs, err
+}
+
+// close a badger database
+func (ss *ServerStorage) close() {
+	ss.db.Close()
+}

client/serverstorage_test.go

@@ -0,0 +1,33 @@
+package client
+
+import (
+	"log"
+	"testing"
+
+	"forge.redroom.link/yves/meowlib"
+)
+
+func TestStoreServer(t *testing.T) {
+	GetConfig().SetMemPass("test")
+	ss := ServerStorage{DbFile: "test.db"}
+	srv := Server{
+		Name:      "test",
+		Url:       "http://127.0.0.1",
+		PublicKey: meowlib.NewKeyPair().Public,
+	}
+	err := ss.StoreServer(&srv)
+	if err != nil {
+		log.Fatal(err)
+	}
+	sout, err := ss.LoadServer(srv.GetServerCard().GetUid())
+	if err != nil {
+		log.Fatal(err)
+	}
+	if sout == nil {
+		log.Fatal("server not found")
+	}
+	if sout.Name != srv.Name {
+		log.Fatal("name not found")
+	}
+
+}

endtoend_test.go

@@ -23,14 +23,19 @@ func TestEndToEnd(t *testing.T) {
 		// Creating New Identity //
 		///////////////////////////
 		Me = client.CreateIdentity("myname")
+
 		// define my preferences (servers)
-		Me.MessageServers.Name = "Message Servers"
-		Me.MessageServers.AddUrls([]string{"http://127.0.0.1/meow/", "mqtt://127.0.0.1", "meow://127.0.0.1"})
+		srv := client.Server{Name: "MyServer", Url: "http://127.0.0.1/meow/"}
+		Me.MessageServers.StoreServer(&srv)
+		srv = client.Server{Name: "MyServer", Url: "mqtt://127.0.0.1"}
+		Me.MessageServers.StoreServer(&srv)
+		srv = client.Server{Name: "MyServer", Url: "meow://127.0.0.1"}
+		Me.MessageServers.StoreServer(&srv)
 		////////////////////////////////////////////////////////////////////////////
 		// Create an invitation for a friend, I want him/her to know me as Bender //
 		////////////////////////////////////////////////////////////////////////////
 		fmt.Println("Creating an invitation for the first friend...")
-		peer, err := Me.InvitePeer("Bender", "myfirstfriend", []int{1, 2}, "welcome, it's me!")
+		peer, err := Me.InvitePeer("Bender", "myfirstfriend", []string{"http://127.0.0.1/meow/", "mqtt://127.0.0.1"}, "welcome, it's me!")
 		if err != nil {
 			println(err)
 		}
@@ -100,9 +105,10 @@ func TestEndToEnd(t *testing.T) {
 		// Packing it
 		packedMsg := MyFirstFriend.PackUserMessage(enc.Data, enc.Signature)
 
-		srv := MyFirstFriend.Contact.PullServers[0]
-		intS1 := client.CreateServerFromServerCard(srv)
-
+		intS1, err := Me.MessageServers.LoadServer("http://127.0.0.1/meow/")
+		if err != nil {
+			fmt.Println(err.Error())
+		}
 		// Creating Server message for transporting the user message
 		toServerMessage, err := intS1.BuildMessageSendingMessage(packedMsg)
 		if err != nil {

servercard.go

@@ -1,10 +1,5 @@
 package meowlib
 
-import (
-	"github.com/dgraph-io/badger"
-	"google.golang.org/protobuf/proto"
-)
-
 func (sc *ServerCard) GetUid() string {
 	return sc.Login + ":" + sc.Password + "@" + sc.Url
 }
@@ -12,101 +7,3 @@ func (sc *ServerCard) GetUid() string {
 func (sc *ServerCard) IsSame(sc1 *ServerCard) bool {
 	return sc.GetUid() == sc1.GetUid()
 }
-
-//
-// Storage
-//
-
-// Open a badger database
-func OpenBadgerDb(path string) (*badger.DB, error) {
-	opts := badger.DefaultOptions(path)
-	opts.Logger = nil
-	return badger.Open(opts)
-}
-
-// Store function stores a ServerCard in a badger database with servercard.GetUid() as key
-func (sc *ServerCard) Store(db *badger.DB) error {
-	// first marchal the ServerCard to bytes with protobuf
-	b, err := proto.Marshal(sc)
-	if err != nil {
-		return err
-	}
-	// then store it in the database
-	return db.Update(func(txn *badger.Txn) error {
-		return txn.Set([]byte(sc.GetUid()), b)
-	})
-}
-
-// LoadServerCard function loads a ServerCard from a badger database with servercard.GetUid() as key
-func LoadServerCard(db *badger.DB, uid string) (*ServerCard, error) {
-	var sc ServerCard
-	err := db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(uid))
-		if err != nil {
-			return err
-		}
-		return item.Value(func(val []byte) error {
-			return proto.Unmarshal(val, &sc)
-		})
-	})
-	return &sc, err
-}
-
-// DeleteServerCard function deletes a ServerCard from a badger database with servercard.GetUid() as key
-func DeleteServerCard(db *badger.DB, uid string) error {
-	return db.Update(func(txn *badger.Txn) error {
-		return txn.Delete([]byte(uid))
-	})
-}
-
-// LoadAllServerCards function loads all ServerCards from a badger database
-func LoadAllServerCards(db *badger.DB) ([]*ServerCard, error) {
-	var scs []*ServerCard
-	err := db.View(func(txn *badger.Txn) error {
-		opts := badger.DefaultIteratorOptions
-		opts.PrefetchSize = 10
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			var sc ServerCard
-			err := item.Value(func(val []byte) error {
-				return proto.Unmarshal(val, &sc)
-			})
-			if err != nil {
-				return err
-			}
-			scs = append(scs, &sc)
-		}
-		return nil
-	})
-	return scs, err
-}
-
-// LoadServerCardsFromUids function loads ServerCards with id in []Uid parameter from a badger database
-func LoadServerCardsFromUids(db *badger.DB, uids []string) ([]*ServerCard, error) {
-	var scs []*ServerCard
-	err := db.View(func(txn *badger.Txn) error {
-		for _, uid := range uids {
-			item, err := txn.Get([]byte(uid))
-			if err != nil {
-				return err
-			}
-			var sc ServerCard
-			err = item.Value(func(val []byte) error {
-				return proto.Unmarshal(val, &sc)
-			})
-			if err != nil {
-				return err
-			}
-			scs = append(scs, &sc)
-		}
-		return nil
-	})
-	return scs, err
-}
-
-// close a badger database
-func CloseBadgerDb(db *badger.DB) {
-	db.Close()
-}