bokoFs/mod/disktool/raid/handler.go

package raid

import (
	"encoding/json"
	"log"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"imuslab.com/bokofs/bokofsd/mod/disktool/diskfs"
	"imuslab.com/bokofs/bokofsd/mod/utils"
)

/*
	Handler.go

	This module handle api call to the raid module
*/

// Handle remove a member disk (sdX) from RAID volume (mdX)
func (m *Manager) HandleRemoveDiskFromRAIDVol(w http.ResponseWriter, r *http.Request) {
	//mdadm --remove /dev/md0 /dev/sdb1
	mdDev, err := utils.PostPara(r, "raidDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid device given")
		return
	}

	sdXDev, err := utils.PostPara(r, "memDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid member device given")
		return
	}

	//Check if target array exists
	if !m.RAIDDeviceExists(mdDev) {
		utils.SendErrorResponse(w, "target RAID array not exists")
		return
	}

	//Check if this is the only disk in the array
	if !m.IsSafeToRemove(mdDev, sdXDev) {
		utils.SendErrorResponse(w, "removal of this device will cause data loss")
		return
	}

	//Check if the disk is already failed
	diskAlreadyFailed, err := m.DiskIsFailed(mdDev, sdXDev)
	if err != nil {
		log.Println("[RAID] Unable to validate if disk failed: " + err.Error())
		utils.SendErrorResponse(w, err.Error())
		return
	}
	//Disk not failed. Mark it as failed
	if !diskAlreadyFailed {
		err = m.FailDisk(mdDev, sdXDev)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
	}

	//Add some delay for OS level to handle IO closing
	time.Sleep(300 * time.Millisecond)

	//Done. Remove the device from array
	err = m.RemoveDisk(mdDev, sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	log.Println("[RAID] Memeber disk " + sdXDev + " removed from RAID volume " + mdDev)
	utils.SendOK(w)
}

// Handle adding a disk (mdX) to RAID volume (mdX)
func (m *Manager) HandleAddDiskToRAIDVol(w http.ResponseWriter, r *http.Request) {
	//mdadm --add /dev/md0 /dev/sdb1
	mdDev, err := utils.PostPara(r, "raidDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid device given")
		return
	}

	sdXDev, err := utils.PostPara(r, "memDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid member device given")
		return
	}

	//Check if target array exists
	if !m.RAIDDeviceExists(mdDev) {
		utils.SendErrorResponse(w, "target RAID array not exists")
		return
	}

	//Check if disk already in another RAID array or mounted
	isMounted, err := diskfs.DeviceIsMounted(sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, "unable to read device state")
		return
	}

	if isMounted {
		utils.SendErrorResponse(w, "target device is mounted")
		return
	}

	diskUsedByAnotherRAID, err := m.DiskIsUsedInAnotherRAIDVol(sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
	}

	if diskUsedByAnotherRAID {
		utils.SendErrorResponse(w, "target device already been used by another RAID volume")
		return
	}

	isOSDisk, err := m.DiskIsRoot(sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
	}

	if isOSDisk {
		utils.SendErrorResponse(w, "OS disk cannot be used as RAID member")
		return
	}

	//OK! Clear the disk
	err = m.ClearSuperblock(sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, "unable to clear superblock of device")
		return
	}

	//Add it to the target RAID array
	err = m.AddDisk(mdDev, sdXDev)
	if err != nil {
		utils.SendErrorResponse(w, "adding disk to RAID volume failed")
		return
	}

	log.Println("[RAID] Device " + sdXDev + " added to RAID volume " + mdDev)

	utils.SendOK(w)
}

// Handle force flush reloading mdadm to solve the md0 become md127 problem
func (m *Manager) HandleMdadmFlushReload(w http.ResponseWriter, r *http.Request) {
	err := m.FlushReload()
	if err != nil {
		utils.SendErrorResponse(w, "reload failed: "+strings.ReplaceAll(err.Error(), "\n", " "))
		return
	}
	utils.SendOK(w)
}

// Handle resolving the disk model label, might return null
func (m *Manager) HandleResolveDiskModelLabel(w http.ResponseWriter, r *http.Request) {
	devName, err := utils.GetPara(r, "devName")
	if err != nil {
		utils.SendErrorResponse(w, "invalid device name given")
		return
	}

	//Function only accept sdX not /dev/sdX
	devName = filepath.Base(devName)

	labelSize, labelModel, err := diskfs.GetDiskModelByName(devName)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	js, _ := json.Marshal([]string{labelModel, labelSize})
	utils.SendJSONResponse(w, string(js))
}

// Handle force flush reloading mdadm to solve the md0 become md127 problem
func (m *Manager) HandlListChildrenDeviceInfo(w http.ResponseWriter, r *http.Request) {
	devName, err := utils.GetPara(r, "devName")
	if err != nil {
		utils.SendErrorResponse(w, "invalid device name given")
		return
	}

	if !strings.HasPrefix(devName, "/dev/") {
		devName = "/dev/" + devName
	}

	//Get the children devices for this RAID
	raidDevice, err := m.GetRAIDDeviceByDevicePath(devName)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Merge the child devices info into one array
	results := map[string]*diskfs.BlockDeviceMeta{}
	for _, blockdevice := range raidDevice.Members {
		bdm, err := diskfs.GetBlockDeviceMeta("/dev/" + blockdevice.Name)
		if err != nil {
			log.Println("[RAID] Unable to load block device info: " + err.Error())
			results[blockdevice.Name] = &diskfs.BlockDeviceMeta{
				Name: blockdevice.Name,
				Size: -1,
			}

			continue
		}

		results[blockdevice.Name] = bdm
	}

	js, _ := json.Marshal(results)
	utils.SendJSONResponse(w, string(js))
}

// Handle list all the disks that is usable
func (m *Manager) HandleListUsableDevices(w http.ResponseWriter, r *http.Request) {
	storageDevices, err := diskfs.ListAllStorageDevices()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Filter out the block devices that are disks
	usableDisks := []diskfs.BlockDeviceMeta{}
	for _, device := range storageDevices.Blockdevices {
		if device.Type == "disk" {
			usableDisks = append(usableDisks, device)
		}
	}

	js, _ := json.Marshal(usableDisks)
	utils.SendJSONResponse(w, string(js))

}

// Handle loading the detail of a given RAID array
func (m *Manager) HandleLoadArrayDetail(w http.ResponseWriter, r *http.Request) {
	devName, err := utils.GetPara(r, "devName")
	if err != nil {
		utils.SendErrorResponse(w, "invalid device name given")
		return
	}

	if !strings.HasPrefix(devName, "/dev/") {
		devName = "/dev/" + devName
	}

	//Check device exists
	if !utils.FileExists(devName) {
		utils.SendErrorResponse(w, "target device not exists")
		return
	}

	//Get status of the array
	targetRAIDInfo, err := m.GetRAIDInfo(devName)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	js, _ := json.Marshal(targetRAIDInfo)
	utils.SendJSONResponse(w, string(js))
}

// Handle formating a device
func (m *Manager) HandleFormatRaidDevice(w http.ResponseWriter, r *http.Request) {
	devName, err := utils.GetPara(r, "devName")
	if err != nil {
		utils.SendErrorResponse(w, "invalid device name given")
		return
	}

	format, err := utils.GetPara(r, "format")
	if err != nil {
		utils.SendErrorResponse(w, "invalid device name given")
		return
	}

	if !strings.HasPrefix(devName, "/dev/") {
		devName = "/dev/" + devName
	}

	//Check if the target device exists
	if !m.RAIDDeviceExists(devName) {
		utils.SendErrorResponse(w, "target not exists or not a valid RAID device")
		return
	}

	//Format the drive
	err = diskfs.FormatStorageDevice(format, devName)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// List all the raid device in this system
func (m *Manager) HandleListRaidDevices(w http.ResponseWriter, r *http.Request) {
	rdevs, err := m.GetRAIDDevicesFromProcMDStat()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	results := []*RAIDInfo{}
	for _, rdev := range rdevs {
		arrayInfo, err := m.GetRAIDInfo("/dev/" + rdev.Name)
		if err != nil {
			continue
		}

		results = append(results, arrayInfo)
	}

	js, _ := json.Marshal(results)
	utils.SendJSONResponse(w, string(js))
}

// Create a RAID storage pool
func (m *Manager) HandleCreateRAIDDevice(w http.ResponseWriter, r *http.Request) {
	devName, err := utils.PostPara(r, "devName")
	if err != nil || devName == "" {
		//Use auto generated one
		devName, err = GetNextAvailableMDDevice()
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
	}
	raidName, err := utils.PostPara(r, "raidName")
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid storage name given")
		return
	}
	raidLevelStr, err := utils.PostPara(r, "level")
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid level given")
		return
	}

	raidDevicesJSON, err := utils.PostPara(r, "raidDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid device array given")
		return
	}

	spareDevicesJSON, err := utils.PostPara(r, "spareDev")
	if err != nil {
		utils.SendErrorResponse(w, "invalid spare device array given")
		return
	}

	//Get if superblock require all zeroed (will also do formating after raid constructed)
	zerosuperblock, err := utils.PostBool(r, "zerosuperblock")
	if err != nil {
		zerosuperblock = false
	}

	//Convert raidDevices and spareDevices ID into string slice
	raidDevices := []string{}
	spareDevices := []string{}

	err = json.Unmarshal([]byte(raidDevicesJSON), &raidDevices)
	if err != nil {
		utils.SendErrorResponse(w, "unable to parse raid device into array")
		return
	}

	err = json.Unmarshal([]byte(spareDevicesJSON), &spareDevices)
	if err != nil {
		utils.SendErrorResponse(w, "unable to parse spare devices into array")
		return
	}

	//Make sure RAID Name do not contain spaces or werid charcters
	if strings.Contains(raidName, " ") {
		utils.SendErrorResponse(w, "raid name cannot contain space")
		return
	}

	//Convert raidLevel to int
	raidLevelStr = strings.TrimPrefix(raidLevelStr, "raid")
	raidLevel, err := strconv.Atoi(raidLevelStr)
	if err != nil {
		utils.SendErrorResponse(w, "invalid raid level given")
		return
	}

	if zerosuperblock {
		//Format each drives
		drivesToZeroblocks := []string{}
		for _, raidDev := range raidDevices {
			if !strings.HasPrefix(raidDev, "/dev/") {
				//Prepend /dev/ to it if not set
				raidDev = filepath.Join("/dev/", raidDev)
			}

			if !utils.FileExists(raidDev) {
				//This disk not found
				utils.SendErrorResponse(w, raidDev+" not found")
				return
			}

			thisDisk := raidDev
			drivesToZeroblocks = append(drivesToZeroblocks, thisDisk)
		}

		for _, spareDev := range spareDevices {
			if !strings.HasPrefix(spareDev, "/dev/") {
				//Prepend /dev/ to it if not set
				spareDev = filepath.Join("/dev/", spareDev)
			}

			if !utils.FileExists(spareDev) {
				//This disk not found
				utils.SendErrorResponse(w, spareDev+" not found")
				return
			}

			thisDisk := spareDev
			drivesToZeroblocks = append(drivesToZeroblocks, thisDisk)
		}

		for _, clearPendingDisk := range drivesToZeroblocks {
			//Format all drives
			log.Println("RAID", "Clearning superblock for disk "+clearPendingDisk, nil)
			err = m.ClearSuperblock(clearPendingDisk)
			if err != nil {
				log.Println("RAID", "Unable to format "+clearPendingDisk+": "+err.Error(), err)
				utils.SendErrorResponse(w, err.Error())
				return
			}
		}
	}

	//Create the RAID device
	err = m.CreateRAIDDevice(devName, raidName, raidLevel, raidDevices, spareDevices)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Update the mdadm config
	err = m.UpdateMDADMConfig()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// Request to reload the RAID manager and scan new / fix missing raid pools
func (m *Manager) HandleRaidDevicesAssemble(w http.ResponseWriter, r *http.Request) {
	err := m.RestartRAIDService()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// Remove a given raid device with its name, USE WITH CAUTION
func (m *Manager) HandleRemoveRaideDevice(w http.ResponseWriter, r *http.Request) {
	//TODO: Add protection and switch to POST
	targetDevice, err := utils.PostPara(r, "raidDev")
	if err != nil {
		utils.SendErrorResponse(w, "target device not given")
		return
	}

	//Check if the raid device exists
	if !m.RAIDDeviceExists(targetDevice) {
		utils.SendErrorResponse(w, "target device not exists")
		return
	}

	//Get the RAID device memeber disks
	targetRAIDDevice, err := m.GetRAIDDeviceByDevicePath(targetDevice)
	if err != nil {
		utils.SendErrorResponse(w, "error occured when trying to load target RAID device info")
		return
	}

	//Check if it is mounted. If yes, unmount it
	if !strings.HasPrefix(targetDevice, "/dev/") {
		targetDevice = filepath.Join("/dev/", targetDevice)
	}

	mounted, err := diskfs.DeviceIsMounted(targetDevice)
	if err != nil {
		log.Println("RAID", "Unmount failed: "+err.Error(), err)
		utils.SendErrorResponse(w, err.Error())
		return
	}

	if mounted {
		log.Println("RAID", targetDevice+" is mounted. Trying to unmount...", nil)
		err = diskfs.UnmountDevice(targetDevice)
		if err != nil {
			log.Println("[RAID] Unmount failed: " + err.Error())
			utils.SendErrorResponse(w, err.Error())
			return
		}

		//Wait for 3 seconds to check if it is still mounted
		counter := 0
		for counter < 3 {
			mounted, _ := diskfs.DeviceIsMounted(targetDevice)
			if mounted {
				//Still not unmounted. Wait for it
				log.Println("RAID", "Device still mounted. Retrying in 1 second", nil)
				counter++
				time.Sleep(1 * time.Second)
			} else {
				break
			}
		}

		//Check if it is still mounted
		mounted, _ = diskfs.DeviceIsMounted(targetDevice)
		if mounted {
			utils.SendErrorResponse(w, "unmount RAID partition failed: device is busy")
			return
		}
	}

	//Give it some time for the raid device to finish umount
	time.Sleep(300 * time.Millisecond)

	//Stop & Remove RAID service on the target device
	err = m.StopRAIDDevice(targetDevice)
	if err != nil {
		log.Println("RAID", "Stop RAID partition failed: "+err.Error(), err)
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Zeroblock the RAID device member disks
	for _, memberDisk := range targetRAIDDevice.Members {
		//Member disk name do not contain full path
		name := memberDisk.Name
		if !strings.HasPrefix(name, "/dev/") {
			name = filepath.Join("/dev/", name)
		}

		err = m.ClearSuperblock(name)
		if err != nil {
			log.Println("RAID", "Unable to clear superblock on device "+name, err)
			continue
		}
	}

	//Update the mdadm config
	err = m.UpdateMDADMConfig()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Done
	utils.SendOK(w)
}

// Force reload all RAID config from file
func (m *Manager) HandleForceAssembleReload(w http.ResponseWriter, r *http.Request) {
	err := m.FlushReload()
	if err != nil {
		log.Println("RAID", "mdadm reload failed: "+err.Error(), err)
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// Grow the raid array to maxmium possible size of the current disks
func (m *Manager) HandleGrowRAIDArray(w http.ResponseWriter, r *http.Request) {
	deviceName, err := utils.PostPara(r, "raidDev")
	if err != nil {
		utils.SendErrorResponse(w, "raid device not given")
		return
	}

	if !m.RAIDDeviceExists(deviceName) {
		utils.SendErrorResponse(w, "target raid device not exists")
		return
	}

	//Check the raid is healthy and ok for expansion
	raidNotHealthy, err := m.RAIDArrayContainsFailedDisks(deviceName)
	if err != nil {
		utils.SendErrorResponse(w, "unable to check health state before expansion")
		return
	}
	if raidNotHealthy {
		utils.SendErrorResponse(w, "expand can only be performed on a healthy array")
		return
	}

	//Expand the raid array
	err = m.GrowRAIDDevice(deviceName)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	err = m.RestartRAIDService()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// HandleRenderOverview List the info and health of all loaded RAID array
func (m *Manager) HandleRenderOverview(w http.ResponseWriter, r *http.Request) {
	//Get all raid device from procmd
	rdevs, err := m.GetRAIDDevicesFromProcMDStat()
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	type RaidHealthOverview struct {
		Name      string
		Status    string
		Level     string
		UsedSize  int64
		TotalSize int64
		IsHealthy bool
	}

	results := []*RaidHealthOverview{}

	//Get RAID Status for each devices
	for _, raidDev := range rdevs {
		//Fill in the basic information
		thisRaidOverview := RaidHealthOverview{
			Name:      raidDev.Name,
			Status:    raidDev.Status,
			Level:     raidDev.Level,
			UsedSize:  -1,
			TotalSize: -1,
			IsHealthy: false,
		}

		//Get health status of RAID
		raidPath := filepath.Join("/dev/", strings.TrimPrefix(raidDev.Name, "/dev/"))
		raidStatus, err := GetRAIDStatus(raidPath)
		if err == nil {
			thisRaidOverview.IsHealthy = raidStatus.isHealthy()
		}

		// Get RAID vol size and info
		raidPartitionSize, err := GetRAIDPartitionSize(raidPath)
		if err == nil {
			thisRaidOverview.TotalSize = raidPartitionSize
		}

		raidUsedSize, err := GetRAIDUsedSize(raidPath)
		if err == nil {
			thisRaidOverview.UsedSize = raidUsedSize
		}

		results = append(results, &thisRaidOverview)
	}

	js, _ := json.Marshal(results)
	utils.SendJSONResponse(w, string(js))
}