commit beba7d3038c0693b37db325069d7693c92dc9118
parent 87b50bd9aea8e6596cd216fc80c39ee60ac99a63
Author: Peter Kosyh <p.kosyh@gmail.com>
Date: Fri, 18 Sep 2020 13:29:17 +0300
db.go commented
Diffstat:
| M | ii/db.go | | | 153 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | ii/log.go | | | 5 | +++++ |
2 files changed, 158 insertions(+), 0 deletions(-)
diff --git a/ii/db.go b/ii/db.go
@@ -1,3 +1,7 @@
+// Database functions.
+// Database is the file with line-bundles: msgid:base64 encoded msg.
+// File db.idx is created and mantained automatically.
+// There is also points.txt, db of users.
package ii
import (
@@ -17,6 +21,13 @@ import (
"time"
)
+// This is index entry. Information about message that is loaded in memory.
+// So, the index could not be very huge.
+// Num: sequence number.
+// Id: MsgId
+// Echo: Echoarea
+// To, From, Repto: message attributes
+// Off: offset to bundle-line in database (in bytes)
type MsgInfo struct {
Num int
Id string
@@ -27,12 +38,21 @@ type MsgInfo struct {
From string
}
+// Index object. Holds List and Hash for all MsgInfo entries
+// FileSize is used to auto reread new entries if it has changed by
+// someone.
type Index struct {
Hash map[string]MsgInfo
List []string
FileSize int64
}
+// Database object. Returns by OpenDB.
+// Idx: Index structure (like dictionary).
+// Name: database name, 'db' by default.
+// Sync: used to syncronize access to DB from goroutines (many readers, one writer).
+// IdxSync: same, but for Index.
+// LockDepth: used for recursive file lock, to avoid conflict between ii-tool and ii-node.
type DB struct {
Path string
Idx Index
@@ -42,6 +62,7 @@ type DB struct {
LockDepth int32
}
+// Utility function. Just append line (text) to file (fn)
func append_file(fn string, text string) error {
f, err := os.OpenFile(fn, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@@ -54,6 +75,10 @@ func append_file(fn string, text string) error {
return nil
}
+// Recursive file lock. Used to avoid conflicts between ii-tool and ii-node.
+// Uses mkdir as atomic operation.
+// Note: dirs created as db.LockPath()
+// 16 sec is limit.
func (db *DB) Lock() bool {
if atomic.AddInt32(&db.LockDepth, 1) > 1 {
return true
@@ -70,6 +95,8 @@ func (db *DB) Lock() bool {
return false
}
+// Recursive file lock: unlock
+// See Lock comment.
func (db *DB) Unlock() {
if atomic.AddInt32(&db.LockDepth, -1) > 0 {
return
@@ -77,14 +104,17 @@ func (db *DB) Unlock() {
os.Remove(db.LockPath())
}
+// Returns path to index file.
func (db *DB) IndexPath() string {
return fmt.Sprintf("%s.idx", db.Path)
}
+// Return path to database itself
func (db *DB) BundlePath() string {
return fmt.Sprintf("%s", db.Path)
}
+// Returns path to lock.
func (db *DB) LockPath() string {
pat := strings.Replace(db.Path, "/", "_", -1)
return fmt.Sprintf("%s/%s-bundle.lock", os.TempDir(), pat)
@@ -92,6 +122,7 @@ func (db *DB) LockPath() string {
// var MaxMsgLen int = 128 * 1024 * 1024
+// This function creates index. It locks.
func (db *DB) CreateIndex() error {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -100,6 +131,9 @@ func (db *DB) CreateIndex() error {
return db._CreateIndex()
}
+
+// Utility to pass all lines of file (path) to fn(line).
+// Stops on EOF or fn returns false.
func FileLines(path string, fn func(string) bool) error {
f, err := os.Open(path)
if err != nil {
@@ -112,6 +146,8 @@ func FileLines(path string, fn func(string) bool) error {
return f_lines(f, fn)
}
+// Internal function to implement FileLines. Works with
+// file by *File object.
func f_lines(f *os.File, fn func(string) bool) error {
reader := bufio.NewReader(f)
for {
@@ -139,6 +175,8 @@ func f_lines(f *os.File, fn func(string) bool) error {
return nil
}
+// Internal function of CreateIndex.
+// Does not lock!
func (db *DB) _CreateIndex() error {
fidx, err := os.OpenFile(db.IndexPath(), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@@ -159,6 +197,8 @@ func (db *DB) _CreateIndex() error {
return true
})
}
+
+// Internal function. Create and open new index.
func (db *DB) _ReopenIndex() (*os.File, error) {
err := db._CreateIndex()
if err != nil {
@@ -170,6 +210,10 @@ func (db *DB) _ReopenIndex() (*os.File, error) {
}
return file, nil
}
+
+// Loads index. If index doesent exists, create and load it.
+// If index was changed, reread tail.
+// This function does lock.
func (db *DB) LoadIndex() error {
db.IdxSync.Lock()
defer db.IdxSync.Unlock()
@@ -258,6 +302,11 @@ func (db *DB) LoadIndex() error {
return nil
}
+// Internal function to Lookup message in loaded index.
+// If idx parameter is true, load and created index.
+// Returns MsgInfo pointer or nil if fails.
+// Does lock!
+// bl: look in blacklisted messages too?
func (db *DB) _Lookup(Id string, bl bool, idx bool) *MsgInfo {
if idx {
if err := db.LoadIndex(); err != nil {
@@ -273,6 +322,8 @@ func (db *DB) _Lookup(Id string, bl bool, idx bool) *MsgInfo {
return &info
}
+// Lookup variant, but without locking.
+// Useful if caller do locking logic himself.
func (db *DB) LookupFast(Id string, bl bool) *MsgInfo {
if Id == "" {
return nil
@@ -280,6 +331,11 @@ func (db *DB) LookupFast(Id string, bl bool) *MsgInfo {
return db._Lookup(Id, bl, false)
}
+// Lookup message in index.
+// Do not search blacklisted messages.
+// Creates/load index if needed.
+// Returns MsgInfo pointer.
+// Does lock!
func (db *DB) Lookup(Id string) *MsgInfo {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -289,6 +345,7 @@ func (db *DB) Lookup(Id string) *MsgInfo {
return db._Lookup(Id, false, true)
}
+// Same as Lookup, but checks in blacklisted messages too
func (db *DB) Exists(Id string) *MsgInfo {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -298,6 +355,10 @@ func (db *DB) Exists(Id string) *MsgInfo {
return db._Lookup(Id, true, true)
}
+// Lookup messages in index.
+// Gets: slice of message ids to get.
+// Returns slice of MsgInfo pointers.
+// Does lock!
func (db *DB) LookupIDS(Ids []string) []*MsgInfo {
var info []*MsgInfo
db.Sync.RLock()
@@ -313,6 +374,10 @@ func (db *DB) LookupIDS(Ids []string) []*MsgInfo {
return info
}
+// Internal function. Gets bundle by message id.
+// If idx is true: load/create index.
+// Returns: msgid:base64 bundle.
+// Does not lock!
func (db *DB) _GetBundle(Id string, idx bool) string {
info := db._Lookup(Id, false, idx)
if info == nil {
@@ -342,6 +407,9 @@ func (db *DB) _GetBundle(Id string, idx bool) string {
return bundle
}
+// Get bundle line by message id from db.
+// Does lock!
+// Loads/create index if needed.
func (db *DB) GetBundle(Id string) string {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -351,6 +419,8 @@ func (db *DB) GetBundle(Id string) string {
return db._GetBundle(Id, true)
}
+// Get decoded message from db by message id.
+// Does lock. Loads/create index if needed.
func (db *DB) Get(Id string) *Msg {
bundle := db.GetBundle(Id)
if bundle == "" {
@@ -363,6 +433,9 @@ func (db *DB) Get(Id string) *Msg {
return m
}
+// Fast varian (w/o locking) of Get.
+// Get decoded message from db by message id.
+// Does NOT lock! Loads/create index if needed.
func (db *DB) GetFast(Id string) *Msg {
bundle := db._GetBundle(Id, false)
if bundle == "" {
@@ -375,6 +448,13 @@ func (db *DB) GetFast(Id string) *Msg {
return m
}
+// Query used to make queries to Index
+// If some field of: Echo, Repto, From, To is not ""
+// fields will be matched with MsgInfo entry (logical AND).
+// If Match function is not nil, this function will be used for matching.
+// Blacklisted: search in blacklisted messages if true.
+// User: authorized access to private areas.
+// Start & Lim: slice of query. For example: -1, 1 -- get last message in db. 0, 1 -- first.
type Query struct {
Echo string
Repto string
@@ -387,6 +467,7 @@ type Query struct {
Match func(mi MsgInfo, q Query) bool
}
+// utility function to add string in front of slice
func prependStr(x []string, y string) []string {
x = append(x, "")
copy(x[1:], x)
@@ -394,6 +475,7 @@ func prependStr(x []string, y string) []string {
return x
}
+// Default match function for queries.
func (db *DB) Match(info MsgInfo, r Query) bool {
if r.Blacklisted {
return info.Off < 0
@@ -424,6 +506,11 @@ func (db *DB) Match(info MsgInfo, r Query) bool {
return true
}
+// Used to get information about echoarea
+// Count: number of messages
+// Topics: number of topics
+// Last: last MsgInfo
+// Msg: last message pointer
type Echo struct {
Name string
Count int
@@ -432,6 +519,12 @@ type Echo struct {
Msg *Msg
}
+// Make query and select Echoes
+// Returns: slice of pointers to Echo.
+// names: if not empty, lookup only in theese echoareas
+// Does lock.
+// Load/create index if needed.
+// Echoes sorted by date of last messages.
func (db *DB) Echoes(names []string, q Query) []*Echo {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -507,6 +600,9 @@ func (db *DB) Echoes(names []string, q Query) []*Echo {
return list
}
+// Make query and retuen ids as slice of strings.
+// Does lock. Can create/load index if needed.
+// r: request, see Query
func (db *DB) SelectIDS(r Query) []string {
var Resp []string
db.Sync.Lock()
@@ -556,6 +652,11 @@ func (db *DB) SelectIDS(r Query) []string {
return Resp
}
+// Internal function. Get slice of MsgInfo pointers
+// and create information about topics.
+// Information returns in form of: [topicid][]ids
+// topic id is the msg id of most old parent in echo
+// ids - is the messages in this topic
func (db *DB) GetTopics(mi []*MsgInfo) map[string][]string {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -599,14 +700,23 @@ func (db *DB) GetTopics(mi []*MsgInfo) map[string][]string {
return topics
}
+// Store decoded message in database
+// If message exists, returns error
func (db *DB) Store(m *Msg) error {
return db._Store(m, false)
}
+// Store decoded message in database
+// even it is exists. So, it's like Edit operation.
+// While index loaded, it got last version of message data.
func (db *DB) Edit(m *Msg) error {
return db._Store(m, true)
}
+// Blacklist decoded message.
+// Blacklisting is adding special tag: access/blacklist and Edit operation
+// to store it in DB. While loading index, blacklisted messages
+// are marked by negative Off field (-1).
func (db *DB) Blacklist(m *Msg) error {
m.Tags.Add("access/blacklist")
return db.Edit(m)
@@ -626,6 +736,7 @@ func (db *DB) Blacklist(m *Msg) error {
// return nil
}
+// Internal function used by Store. See Store comment.
func (db *DB) _Store(m *Msg, edit bool) error {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -661,6 +772,9 @@ func (db *DB) _Store(m *Msg, edit bool) error {
return nil
}
+// Opens DB and returns pointer to DB object.
+// path is the path to db. By default it is ./db
+// Index will be named as path + ".idx"
func OpenDB(path string) *DB {
var db DB
db.Path = path
@@ -673,6 +787,9 @@ func OpenDB(path string) *DB {
return &db
}
+// User entry in points.txt db
+// User with Id == 1 is superuser.
+// Tags: custom information (like avatars :) in Tags format
type User struct {
Id int32
Name string
@@ -681,6 +798,12 @@ type User struct {
Tags Tags
}
+// User database.
+// FileSize - size of points.txt to detect DB changes.
+// Names: holds User structure by user name
+// ById: holds user name by user id
+// Secrets: holds user name by user secret (pauth)
+// List: holds user names as list
type UDB struct {
Path string
Names map[string]User
@@ -691,6 +814,7 @@ type UDB struct {
FileSize int64
}
+// Check username if it is valid
func IsUsername(u string) bool {
return !strings.ContainsAny(u, ":\n\r\t/") &&
!strings.HasPrefix(u, " ") &&
@@ -698,16 +822,20 @@ func IsUsername(u string) bool {
len(u) <= 16 && len(u) > 2
}
+// Check password if it is valid to be used
func IsPassword(u string) bool {
return len(u) >= 1
}
+// Make secret from string.
+// String is something like id + user + password
func MakeSecret(msg string) string {
h := sha256.Sum256([]byte(msg))
s := base64.URLEncoding.EncodeToString(h[:])
return s[0:10]
}
+// Return secret for username or "" if no such user
func (db *UDB) Secret(User string) string {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -718,6 +846,7 @@ func (db *UDB) Secret(User string) string {
return ui.Secret
}
+// Returns true if user+password is valid
func (db *UDB) Auth(User string, Passwd string) bool {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -728,6 +857,7 @@ func (db *UDB) Auth(User string, Passwd string) bool {
return ui.Secret == MakeSecret(User+Passwd)
}
+// Returns true if Secret (pauth) is valid
func (db *UDB) Access(Secret string) bool {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -735,6 +865,7 @@ func (db *UDB) Access(Secret string) bool {
return ok
}
+// Return username for given Secret
func (db *UDB) Name(Secret string) string {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -746,6 +877,7 @@ func (db *UDB) Name(Secret string) string {
return ""
}
+// Return User pointer for given Secret
func (db *UDB) UserInfo(Secret string) *User {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -757,6 +889,8 @@ func (db *UDB) UserInfo(Secret string) *User {
Error.Printf("No user for secret: %s", Secret)
return nil
}
+
+// Return User pointer for user id
func (db *UDB) UserInfoId(id int32) *User {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -768,6 +902,8 @@ func (db *UDB) UserInfoId(id int32) *User {
Error.Printf("No user for Id: %d", id)
return nil
}
+
+// Return User pointer for given user name
func (db *UDB) UserInfoName(name string) *User {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -778,6 +914,7 @@ func (db *UDB) UserInfoName(name string) *User {
return nil
}
+// Return user id for given secret
func (db *UDB) Id(Secret string) int32 {
db.Sync.RLock()
defer db.Sync.RUnlock()
@@ -795,6 +932,8 @@ func (db *UDB) Id(Secret string) int32 {
var emailRegex = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
+// Add (register) user in database
+// Mail is optional but someday it will be used in registration process
func (db *UDB) Add(Name string, Mail string, Passwd string) error {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -831,11 +970,16 @@ func (db *UDB) Add(Name string, Mail string, Passwd string) error {
return nil
}
+// Open user database and return pointer to UDB object
func OpenUsers(path string) *UDB {
var db UDB
db.Path = path
return &db
}
+
+// Change (replace) information about user.
+// Gets pointer to User object and write it in DB, replacing old information.
+// Works atomically using rename.
func (db *UDB) Edit(u *User) error {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -858,6 +1002,8 @@ func (db *UDB) Edit(u *User) error {
return nil
}
+// Load user information in memory if it is needed (FileSize changed).
+// So, it is safe to call it on every request.
func (db *UDB) LoadUsers() error {
db.Sync.Lock()
defer db.Sync.Unlock()
@@ -915,12 +1061,16 @@ func (db *UDB) LoadUsers() error {
return nil
}
+// Echo database entry
+// Holds echo descriptions in Info hash.
+// List - names of echoareas.
type EDB struct {
Info map[string]string
List []string
Path string
}
+// Check if echo is exists in echo database
func (db *EDB) Allowed(name string) bool {
if len(db.List) == 0 {
return true
@@ -930,6 +1080,9 @@ func (db *EDB) Allowed(name string) bool {
}
return false
}
+
+// Loads echolist database and returns pointer to EDB
+// Supposed to be called only once
func LoadEcholist(path string) *EDB {
var db EDB
db.Path = path
diff --git a/ii/log.go b/ii/log.go
@@ -1,3 +1,4 @@
+// Simple log system.
package ii
import (
@@ -12,11 +13,15 @@ var (
Error *log.Logger
)
+// Default mode. All messages are shown.
func InitLog() {
Trace = log.New(os.Stdout, "=== ", log.Ldate|log.Ltime)
Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime)
Error = log.New(os.Stderr, "ERR: ", log.Ldate|log.Ltime)
}
+
+// Custom mode. Use io.Writers to select what verbose level is needed.
+// For example: ii.OpenLog(ioutil.Discard, os.Stdout, os.Stderr)
func OpenLog(trace io.Writer, info io.Writer, error io.Writer) {
Trace = log.New(trace, "=== ", log.Ldate|log.Ltime)
Info = log.New(info, "INFO: ", log.Ldate|log.Ltime)
