Finalized fsdb implementation for openwrt

.gitignore

@@ -19,6 +19,7 @@ system/ao.db.lock
 *aofs.db.lock
 ./web/aofs.db
 system/auth/authlog.db
+fsdb/*
 
 #Setting related
 system/network/wifi/ap/*

mod/database/database.go

@@ -9,38 +9,17 @@ package database
 */
 
 import (
-	"encoding/json"
-	"errors"
-	"log"
 	"sync"
-
-	"github.com/boltdb/bolt"
 )
 
 type Database struct {
-	Db       *bolt.DB
+	Db       interface{}
 	Tables   sync.Map
 	ReadOnly bool
 }
 
 func NewDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
-	db, err := bolt.Open(dbfile, 0600, nil)
-	log.Println("Key-value Database Service Started: " + dbfile)
-
-	tableMap := sync.Map{}
-	//Build the table list from database
-	err = db.View(func(tx *bolt.Tx) error {
-		return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
-			tableMap.Store(string(name), "")
-			return nil
-		})
-	})
-
-	return &Database{
-		Db:       db,
-		Tables:   tableMap,
-		ReadOnly: readOnlyMode,
-	}, err
+	return newDatabase(dbfile, readOnlyMode)
 }
 
 /*
@@ -56,63 +35,22 @@ func (d *Database) UpdateReadWriteMode(readOnly bool) {
 
 //Dump the whole db into a log file
 func (d *Database) Dump(filename string) ([]string, error) {
-	results := []string{}
-
-	d.Tables.Range(func(tableName, v interface{}) bool {
-		entries, err := d.ListTable(tableName.(string))
-		if err != nil {
-			log.Println("Reading table " + tableName.(string) + " failed: " + err.Error())
-			return false
-		}
-		for _, keypairs := range entries {
-			results = append(results, string(keypairs[0])+":"+string(keypairs[1])+"\n")
-		}
-		return true
-	})
-
-	return results, nil
+	return d.dump(filename)
 }
 
 //Create a new table
 func (d *Database) NewTable(tableName string) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-
-	d.Tables.Store(tableName, "")
-	return err
+	return d.newTable(tableName)
 }
 
 //Check is table exists
 func (d *Database) TableExists(tableName string) bool {
-	if _, ok := d.Tables.Load(tableName); ok {
-		return true
-	}
-	return false
+	return d.tableExists(tableName)
 }
 
 //Drop the given table
 func (d *Database) DropTable(tableName string) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.Update(func(tx *bolt.Tx) error {
-		err := tx.DeleteBucket([]byte(tableName))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	return err
+	return d.dropTable(tableName)
 }
 
 /*
@@ -126,21 +64,7 @@ func (d *Database) DropTable(tableName string) error {
 	err := sysdb.Write("MyTable", "username/message",thisDemo);
 */
 func (d *Database) Write(tableName string, key string, value interface{}) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	jsonString, err := json.Marshal(value)
-	if err != nil {
-		return err
-	}
-	err = d.Db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
-		b := tx.Bucket([]byte(tableName))
-		err = b.Put([]byte(key), jsonString)
-		return err
-	})
-	return err
+	return d.write(tableName, key, value)
 }
 
 /*
@@ -154,40 +78,11 @@ func (d *Database) Write(tableName string, key string, value interface{}) error
 */
 
 func (d *Database) Read(tableName string, key string, assignee interface{}) error {
-	err := d.Db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		v := b.Get([]byte(key))
-		json.Unmarshal(v, &assignee)
-		return nil
-	})
-	return err
+	return d.read(tableName, key, assignee)
 }
 
 func (d *Database) KeyExists(tableName string, key string) bool {
-	resultIsNil := false
-	if !d.TableExists(tableName) {
-		//Table not exists. Do not proceed accessing key
-		log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
-		return false
-	}
-	err := d.Db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		v := b.Get([]byte(key))
-		if v == nil {
-			resultIsNil = true
-		}
-		return nil
-	})
-
-	if err != nil {
-		return false
-	} else {
-		if resultIsNil {
-			return false
-		} else {
-			return true
-		}
-	}
+	return d.keyExists(tableName, key)
 }
 
 /*
@@ -196,16 +91,7 @@ func (d *Database) KeyExists(tableName string, key string) bool {
 	err := sysdb.Delete("MyTable", "username/message");
 */
 func (d *Database) Delete(tableName string, key string) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.Update(func(tx *bolt.Tx) error {
-		tx.Bucket([]byte(tableName)).Delete([]byte(key))
-		return nil
-	})
-
-	return err
+	return d.delete(tableName, key)
 }
 
 /*
@@ -226,20 +112,9 @@ func (d *Database) Delete(tableName string, key string) error {
 */
 
 func (d *Database) ListTable(tableName string) ([][][]byte, error) {
-	var results [][][]byte
-	err := d.Db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		c := b.Cursor()
-
-		for k, v := c.First(); k != nil; k, v = c.Next() {
-			results = append(results, [][]byte{k, v})
-		}
-		return nil
-	})
-	return results, err
+	return d.listTable(tableName)
 }
 
 func (d *Database) Close() {
-	d.Db.Close()
-	return
+	d.close()
 }

mod/database/database_core.go

@@ -0,0 +1,188 @@
+//go:build !mipsle
+// +build !mipsle
+
+package database
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"sync"
+
+	"github.com/boltdb/bolt"
+)
+
+func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
+	db, err := bolt.Open(dbfile, 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Println("Key-value Database Service Started: " + dbfile)
+
+	tableMap := sync.Map{}
+	//Build the table list from database
+	err = db.View(func(tx *bolt.Tx) error {
+		return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
+			tableMap.Store(string(name), "")
+			return nil
+		})
+	})
+
+	return &Database{
+		Db:       db,
+		Tables:   tableMap,
+		ReadOnly: readOnlyMode,
+	}, err
+}
+
+//Dump the whole db into a log file
+func (d *Database) dump(filename string) ([]string, error) {
+	results := []string{}
+
+	d.Tables.Range(func(tableName, v interface{}) bool {
+		entries, err := d.ListTable(tableName.(string))
+		if err != nil {
+			log.Println("Reading table " + tableName.(string) + " failed: " + err.Error())
+			return false
+		}
+		for _, keypairs := range entries {
+			results = append(results, string(keypairs[0])+":"+string(keypairs[1])+"\n")
+		}
+		return true
+	})
+
+	return results, nil
+}
+
+//Create a new table
+func (d *Database) newTable(tableName string) error {
+	if d.ReadOnly == true {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	d.Tables.Store(tableName, "")
+	return err
+}
+
+//Check is table exists
+func (d *Database) tableExists(tableName string) bool {
+	if _, ok := d.Tables.Load(tableName); ok {
+		return true
+	}
+	return false
+}
+
+//Drop the given table
+func (d *Database) dropTable(tableName string) error {
+	if d.ReadOnly == true {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		err := tx.DeleteBucket([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}
+
+//Write to table
+func (d *Database) write(tableName string, key string, value interface{}) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+
+	jsonString, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		b := tx.Bucket([]byte(tableName))
+		err = b.Put([]byte(key), jsonString)
+		return err
+	})
+	return err
+}
+
+func (d *Database) read(tableName string, key string, assignee interface{}) error {
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		v := b.Get([]byte(key))
+		json.Unmarshal(v, &assignee)
+		return nil
+	})
+	return err
+}
+
+func (d *Database) keyExists(tableName string, key string) bool {
+	resultIsNil := false
+	if !d.TableExists(tableName) {
+		//Table not exists. Do not proceed accessing key
+		log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
+		return false
+	}
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		v := b.Get([]byte(key))
+		if v == nil {
+			resultIsNil = true
+		}
+		return nil
+	})
+
+	if err != nil {
+		return false
+	} else {
+		if resultIsNil {
+			return false
+		} else {
+			return true
+		}
+	}
+}
+
+func (d *Database) delete(tableName string, key string) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		tx.Bucket([]byte(tableName)).Delete([]byte(key))
+		return nil
+	})
+
+	return err
+}
+
+func (d *Database) listTable(tableName string) ([][][]byte, error) {
+	var results [][][]byte
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		c := b.Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			results = append(results, [][]byte{k, v})
+		}
+		return nil
+	})
+	return results, err
+}
+
+func (d *Database) close() {
+	d.Db.(*bolt.DB).Close()
+}

mod/database/database_openwrt.go

@@ -0,0 +1,208 @@
+//go:build mipsle
+
+package database
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
+	dbRootPath := filepath.ToSlash(filepath.Clean(dbfile))
+	dbRootPath = "fsdb/" + dbRootPath
+	err := os.MkdirAll(dbRootPath, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	tableMap := sync.Map{}
+	//build the table list from file system
+	files, err := filepath.Glob(filepath.Join(dbRootPath, "/*"))
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		if isDirectory(file) {
+			tableMap.Store(filepath.Base(file), "")
+		}
+	}
+
+	log.Println("Filesystem Emulated Key-value Database Service Started: " + dbRootPath)
+	return &Database{
+		Db:       dbRootPath,
+		Tables:   tableMap,
+		ReadOnly: readOnlyMode,
+	}, nil
+}
+
+func (d *Database) dump(filename string) ([]string, error) {
+	//Get all file objects from root
+	rootfiles, err := filepath.Glob(filepath.Join(d.Db.(string), "/*"))
+	if err != nil {
+		return []string{}, err
+	}
+
+	//Filter out the folders
+	rootFolders := []string{}
+	for _, file := range rootfiles {
+		if !isDirectory(file) {
+			rootFolders = append(rootFolders, filepath.Base(file))
+		}
+	}
+
+	return rootFolders, nil
+}
+
+func (d *Database) newTable(tableName string) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	if !fileExists(tablePath) {
+		return os.MkdirAll(tablePath, 0755)
+	}
+	return nil
+}
+
+func (d *Database) tableExists(tableName string) bool {
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	if _, err := os.Stat(tablePath); errors.Is(err, os.ErrNotExist) {
+		return false
+	}
+
+	if !isDirectory(tablePath) {
+		return false
+	}
+
+	return true
+}
+
+func (d *Database) dropTable(tableName string) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	if d.tableExists(tableName) {
+		return os.RemoveAll(tablePath)
+	} else {
+		return errors.New("table not exists")
+	}
+
+}
+
+func (d *Database) write(tableName string, key string, value interface{}) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	js, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+
+	key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
+
+	return ioutil.WriteFile(filepath.Join(tablePath, key+".entry"), js, 0755)
+}
+
+func (d *Database) read(tableName string, key string, assignee interface{}) error {
+	if !d.keyExists(tableName, key) {
+		return errors.New("key not exists")
+	}
+
+	key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
+
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	entryPath := filepath.Join(tablePath, key+".entry")
+	content, err := ioutil.ReadFile(entryPath)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(content, &assignee)
+	return err
+}
+
+func (d *Database) keyExists(tableName string, key string) bool {
+	key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	entryPath := filepath.Join(tablePath, key+".entry")
+	return fileExists(entryPath)
+}
+
+func (d *Database) delete(tableName string, key string) error {
+	if d.ReadOnly {
+		return errors.New("Operation rejected in ReadOnly mode")
+	}
+	if !d.keyExists(tableName, key) {
+		return errors.New("key not exists")
+	}
+	key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	entryPath := filepath.Join(tablePath, key+".entry")
+
+	return os.Remove(entryPath)
+}
+
+func (d *Database) listTable(tableName string) ([][][]byte, error) {
+	if !d.tableExists(tableName) {
+		return [][][]byte{}, errors.New("table not exists")
+	}
+	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
+	entries, err := filepath.Glob(filepath.Join(tablePath, "/*.entry"))
+	if err != nil {
+		return [][][]byte{}, err
+	}
+
+	var results [][][]byte = [][][]byte{}
+	for _, entry := range entries {
+		if !isDirectory(entry) {
+			//Read it
+			key := filepath.Base(entry)
+			key = strings.TrimSuffix(key, filepath.Ext(key))
+			key = strings.ReplaceAll(key, "-SLASH_SIGN-", "/")
+
+			bkey := []byte(key)
+			bval := []byte("")
+			c, err := ioutil.ReadFile(entry)
+			if err != nil {
+				break
+			}
+
+			bval = c
+			results = append(results, [][]byte{bkey, bval})
+		}
+	}
+	return results, nil
+}
+
+func (d *Database) close() {
+	//Nothing to close as it is file system
+}
+
+func isDirectory(path string) bool {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return fileInfo.IsDir()
+}
+
+func fileExists(name string) bool {
+	_, err := os.Stat(name)
+	if err == nil {
+		return true
+	}
+	if errors.Is(err, os.ErrNotExist) {
+		return false
+	}
+	return false
+}

openwrt-build.bat

@@ -0,0 +1,7 @@
+echo Building openWRT
+set GOARCH=mipsle
+set GOOS=linux
+set GOMIPS=softfloat
+set CGO_ENABLED=0
+go build -ldflags "-s -w" -trimpath
+ren "arozos" "arozos_linux_mipsle"