Added experimental new upload method for FORM POST

file_system.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"log"
 	"math"
-	"mime/multipart"
 	"net/http"
 	"net/url"
 	"os"
@@ -27,11 +26,11 @@ import (
 	fsp "imuslab.com/arozos/mod/filesystem/fspermission"
 	hidden "imuslab.com/arozos/mod/filesystem/hidden"
 	metadata "imuslab.com/arozos/mod/filesystem/metadata"
+	"imuslab.com/arozos/mod/filesystem/upload"
 	module "imuslab.com/arozos/mod/modules"
 	prout "imuslab.com/arozos/mod/prouter"
 	"imuslab.com/arozos/mod/share"
 	storage "imuslab.com/arozos/mod/storage"
-	user "imuslab.com/arozos/mod/user"
 )
 
 var (
@@ -524,75 +523,134 @@ func system_fs_handleUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = r.ParseMultipartForm(int64(*upload_buf) << 20)
+	uploadedFilepaths, err := upload.StreamUploadToDisk(userinfo, w, r)
 	if err != nil {
-		//Filesize too big
-		log.Println(err)
-		sendErrorResponse(w, "File too large")
+		sendErrorResponse(w, err.Error())
 		return
 	}
 
-	file, handler, err := r.FormFile("file")
-	if err != nil {
-		log.Println("Error Retrieving File from upload by user: " + username)
-		sendErrorResponse(w, "Unable to parse file from upload")
-		return
+	//Set the ownership of file(s)
+	quotaFulled := false
+	for _, fileRealpath := range uploadedFilepaths {
+		//Check for storage quota
+		uploadFileSize := fs.GetFileSize(fileRealpath)
+		if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
+			//User storage quota fulled. Remove this file
+			quotaFulled = true
+
+			//Remove this file as this doesn't fit
+			os.Remove(fileRealpath)
+		} else {
+			userinfo.SetOwnerOfFile(fileRealpath)
+		}
 	}
 
-	//Get upload target directory
-	uploadTarget, _ := mv(r, "path", true)
-	if uploadTarget == "" {
-		sendErrorResponse(w, "Upload target cannot be empty.")
+	if quotaFulled {
+		sendErrorResponse(w, "Storage Quota Full")
 		return
 	}
 
-	//Translate the upload target directory
-	realUploadPath, err := userinfo.VirtualPathToRealPath(uploadTarget)
+	sendOK(w)
+	return
 
-	if err != nil {
-		sendErrorResponse(w, "Upload target is invalid or permission denied.")
-		return
-	}
+	/*
+		err = r.ParseMultipartForm(int64(*upload_buf) << 20)
+		if err != nil {
+			//Filesize too big
+			log.Println(err)
+			sendErrorResponse(w, "File too large")
+			return
+		}
 
-	storeFilename := handler.Filename //Filename of the uploaded file
-	destFilepath := filepath.ToSlash(filepath.Clean(realUploadPath)) + "/" + storeFilename
+		file, handler, err := r.FormFile("file")
+		if err != nil {
+			log.Println("Error Retrieving File from upload by user: " + username)
+			sendErrorResponse(w, "Unable to parse file from upload")
+			return
+		}
 
-	if !fileExists(filepath.Dir(destFilepath)) {
-		os.MkdirAll(filepath.Dir(destFilepath), 0755)
-	}
+		//Get upload target directory
+		uploadTarget, _ := mv(r, "path", true)
+		if uploadTarget == "" {
+			sendErrorResponse(w, "Upload target cannot be empty.")
+			return
+		}
 
-	//Check if the upload target is read only.
-	accmode := userinfo.GetPathAccessPermission(uploadTarget)
-	if accmode == "readonly" {
-		sendErrorResponse(w, "The upload target is Read Only.")
-		return
-	} else if accmode == "denied" {
-		sendErrorResponse(w, "Access Denied")
-		return
-	}
+		//Translate the upload target directory
+		realUploadPath, err := userinfo.VirtualPathToRealPath(uploadTarget)
+		if err != nil {
+			sendErrorResponse(w, "Upload target is invalid or permission denied.")
+			return
+		}
 
-	//Check for storage quota
-	uploadFileSize := handler.Size
-	if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
-		sendErrorResponse(w, "Storage Quota Full")
-		return
-	}
+		storeFilename := handler.Filename //Filename of the uploaded file
+		destFilepath := filepath.ToSlash(filepath.Clean(realUploadPath)) + "/" + storeFilename
 
-	//Prepare the file to be created (uploaded)
-	destination, err := os.Create(destFilepath)
-	if err != nil {
-		sendErrorResponse(w, err.Error())
-		return
-	}
+		if !fileExists(filepath.Dir(destFilepath)) {
+			os.MkdirAll(filepath.Dir(destFilepath), 0755)
+		}
 
-	defer destination.Close()
-	defer file.Close()
+		//Check if the upload target is read only.
+		accmode := userinfo.GetPathAccessPermission(uploadTarget)
+		if accmode == "readonly" {
+			sendErrorResponse(w, "The upload target is Read Only.")
+			return
+		} else if accmode == "denied" {
+			sendErrorResponse(w, "Access Denied")
+			return
+		}
+
+		//Check for storage quota
+		uploadFileSize := handler.Size
+		if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
+			sendErrorResponse(w, "Storage Quota Full")
+			return
+		}
+
+		//Prepare the file to be created (uploaded)
+		destination, err := os.Create(destFilepath)
+		if err != nil {
+			sendErrorResponse(w, err.Error())
+			return
+		}
+
+		defer destination.Close()
+		defer file.Close()
+
+		//Move the file to destination file location
+		if *enable_asyncFileUpload {
+			//Use Async upload method
+			go func(r *http.Request, file multipart.File, destination *os.File, userinfo *user.User) {
+				//Do the file copying using a buffered reader
+				buf := make([]byte, *file_opr_buff)
+				for {
+					n, err := file.Read(buf)
+					if err != nil && err != io.EOF {
+						log.Println(err.Error())
+						return
+					}
+					if n == 0 {
+						break
+					}
+
+					if _, err := destination.Write(buf[:n]); err != nil {
+						log.Println(err.Error())
+						return
+					}
+				}
+
+				//Clear up buffered files
+				r.MultipartForm.RemoveAll()
+
+				//Set the ownership of file
+				userinfo.SetOwnerOfFile(destFilepath)
 
-	//Move the file to destination file location
-	if *enable_asyncFileUpload {
-		//Use Async upload method
-		go func(r *http.Request, file multipart.File, destination *os.File, userinfo *user.User) {
-			//Do the file copying using a buffered reader
+				//Perform a GC afterward
+				runtime.GC()
+
+			}(r, file, destination, userinfo)
+		} else {
+			//Use blocking upload and move method
 			buf := make([]byte, *file_opr_buff)
 			for {
 				n, err := file.Read(buf)
@@ -615,55 +673,27 @@ func system_fs_handleUpload(w http.ResponseWriter, r *http.Request) {
 
 			//Set the ownership of file
 			userinfo.SetOwnerOfFile(destFilepath)
-
-			//Perform a GC afterward
-			runtime.GC()
-
-		}(r, file, destination, userinfo)
-	} else {
-		//Use blocking upload and move method
-		buf := make([]byte, *file_opr_buff)
-		for {
-			n, err := file.Read(buf)
-			if err != nil && err != io.EOF {
-				log.Println(err.Error())
-				return
-			}
-			if n == 0 {
-				break
-			}
-
-			if _, err := destination.Write(buf[:n]); err != nil {
-				log.Println(err.Error())
-				return
-			}
 		}
 
-		//Clear up buffered files
-		r.MultipartForm.RemoveAll()
-
-		//Set the ownership of file
-		userinfo.SetOwnerOfFile(destFilepath)
-	}
-
-	//Finish up the upload
+		//Finish up the upload
 
-	//fmt.Printf("Uploaded File: %+v\n", handler.Filename)
-	//fmt.Printf("File Size: %+v\n", handler.Size)
-	//fmt.Printf("MIME Header: %+v\n", handler.Header)
-	//fmt.Println("Upload target: " + realUploadPath)
+		//fmt.Printf("Uploaded File: %+v\n", handler.Filename)
+		//fmt.Printf("File Size: %+v\n", handler.Size)
+		//fmt.Printf("MIME Header: %+v\n", handler.Header)
+		//fmt.Println("Upload target: " + realUploadPath)
 
-	//Fnish upload. Fix the tmp filename
-	log.Println(username + " uploaded a file: " + handler.Filename)
+		//Fnish upload. Fix the tmp filename
+		log.Println(username + " uploaded a file: " + handler.Filename)
 
-	//Do upload finishing stuff
-	//Perform a GC
-	runtime.GC()
+		//Do upload finishing stuff
+		//Perform a GC
+		runtime.GC()
 
-	//Completed
-	sendOK(w)
+		//Completed
+		sendOK(w)
 
-	return
+		return
+	*/
 }
 
 //Validate if the copy and target process will involve file overwriting problem.

mod/filesystem/upload/common.go

@@ -0,0 +1,161 @@
+package upload
+
+import (
+	"bufio"
+	"encoding/base64"
+	"errors"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+/*
+	Basic Response Functions
+
+	Send response with ease
+*/
+//Send text response with given w and message as string
+func sendTextResponse(w http.ResponseWriter, msg string) {
+	w.Write([]byte(msg))
+}
+
+//Send JSON response, with an extra json header
+func sendJSONResponse(w http.ResponseWriter, json string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Write([]byte(json))
+}
+
+func sendErrorResponse(w http.ResponseWriter, errMsg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Write([]byte("{\"error\":\"" + errMsg + "\"}"))
+}
+
+func sendOK(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Write([]byte("\"OK\""))
+}
+
+/*
+	The paramter move function (mv)
+
+	You can find similar things in the PHP version of ArOZ Online Beta. You need to pass in
+	r (HTTP Request Object)
+	getParamter (string, aka $_GET['This string])
+
+	Will return
+	Paramter string (if any)
+	Error (if error)
+
+*/
+func mv(r *http.Request, getParamter string, postMode bool) (string, error) {
+	if postMode == false {
+		//Access the paramter via GET
+		keys, ok := r.URL.Query()[getParamter]
+
+		if !ok || len(keys[0]) < 1 {
+			//log.Println("Url Param " + getParamter +" is missing")
+			return "", errors.New("GET paramter " + getParamter + " not found or it is empty")
+		}
+
+		// Query()["key"] will return an array of items,
+		// we only want the single item.
+		key := keys[0]
+		return string(key), nil
+	} else {
+		//Access the parameter via POST
+		r.ParseForm()
+		x := r.Form.Get(getParamter)
+		if len(x) == 0 || x == "" {
+			return "", errors.New("POST paramter " + getParamter + " not found or it is empty")
+		}
+		return string(x), nil
+	}
+
+}
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+func fileExists(filename string) bool {
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func isDir(path string) bool {
+	if fileExists(path) == false {
+		return false
+	}
+	fi, err := os.Stat(path)
+	if err != nil {
+		log.Fatal(err)
+		return false
+	}
+	switch mode := fi.Mode(); {
+	case mode.IsDir():
+		return true
+	case mode.IsRegular():
+		return false
+	}
+	return false
+}
+
+func inArray(arr []string, str string) bool {
+	for _, a := range arr {
+		if a == str {
+			return true
+		}
+	}
+	return false
+}
+
+func timeToString(targetTime time.Time) string {
+	return targetTime.Format("2006-01-02 15:04:05")
+}
+
+func loadImageAsBase64(filepath string) (string, error) {
+	if !fileExists(filepath) {
+		return "", errors.New("File not exists")
+	}
+	f, _ := os.Open(filepath)
+	reader := bufio.NewReader(f)
+	content, _ := ioutil.ReadAll(reader)
+	encoded := base64.StdEncoding.EncodeToString(content)
+	return string(encoded), nil
+}
+
+func pushToSliceIfNotExist(slice []string, newItem string) []string {
+	itemExists := false
+	for _, item := range slice {
+		if item == newItem {
+			itemExists = true
+		}
+	}
+
+	if !itemExists {
+		slice = append(slice, newItem)
+	}
+
+	return slice
+}
+
+func removeFromSliceIfExists(slice []string, target string) []string {
+	newSlice := []string{}
+	for _, item := range slice {
+		if item != target {
+			newSlice = append(newSlice, item)
+		}
+	}
+
+	return newSlice
+}

mod/filesystem/upload/upload.go

@@ -0,0 +1,121 @@
+package upload
+
+import (
+	"errors"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	user "imuslab.com/arozos/mod/user"
+)
+
+type chunk struct {
+	PartFilename string
+	DestFilename string
+}
+
+func StreamUploadToDisk(userinfo *user.User, w http.ResponseWriter, r *http.Request) ([]string, error) {
+	//Check if this userinfo is valid
+	if userinfo == nil {
+		return []string{}, errors.New("Invalid userinfo")
+	}
+
+	vpath, ok := r.URL.Query()["path"]
+	if !ok || len(vpath) == 0 {
+		return []string{}, errors.New("Invalid upload destination")
+	}
+
+	//Get the upload destination realpath
+	realUploadPath, err := userinfo.VirtualPathToRealPath(vpath[0])
+	if err != nil {
+		//Return virtual path to real path translation error
+		return []string{}, err
+	}
+
+	//Try to parse the FORM POST using multipart reader
+	reader, err := r.MultipartReader()
+	if err != nil {
+		log.Println("Upload failed: " + err.Error())
+		return []string{}, err
+	}
+
+	//Start write process
+	uplaodedFiles := map[string]string{}
+	for {
+		part, err := reader.NextPart()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			//Connection lost when uploading. Remove the uploading file.
+			clearFailedUploadChunks(uplaodedFiles)
+			return []string{}, errors.New("Upload failed")
+		}
+		defer part.Close()
+
+		//Check if this is file or other paramters
+		if part.FileName() != "" {
+			//This part is a part of a file. Write it to destination folder
+			tmpFilepath := filepath.Join(realUploadPath, part.FileName()+".tmp")
+
+			//Check if this part is uploaded before. If not but the .tmp file exists
+			//This is from previous unsuccessful upload and it should be reoved
+			_, ok := uplaodedFiles[part.FileName()]
+			if !ok && fileExists(tmpFilepath) {
+				//This chunk is first chunk of the file and the .tmp file already exists.
+				//Remove it
+				log.Println("Removing previous failed upload: ", tmpFilepath)
+				os.Remove(tmpFilepath)
+			}
+
+			//Check if the uploading target folder exists. If not, create it
+			if !fileExists(filepath.Dir(tmpFilepath)) {
+				os.MkdirAll(filepath.Dir(tmpFilepath), 0755)
+			}
+
+			//Open the file and write to it using append mode
+			d, err := os.OpenFile(tmpFilepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0775)
+			if err != nil {
+				//Failed to create new file.
+				clearFailedUploadChunks(uplaodedFiles)
+				return []string{}, errors.New("Write to disk failed")
+			}
+			io.Copy(d, part)
+			d.Close()
+
+			//Record this file
+			uplaodedFiles[part.FileName()] = tmpFilepath
+		} else {
+			//Unknown stuffs
+			continue
+		}
+
+	}
+
+	//Remove the .tmp extension for the file.
+	uploadedFilepaths := []string{}
+	for thisFilename, thisFilepath := range uplaodedFiles {
+		thisFilename = filepath.Base(thisFilename)
+		finalDestFilename := filepath.Join(filepath.Dir(thisFilepath), thisFilename)
+
+		//Remove the .tmp from the upload by renaming it as the original name
+		err = os.Rename(thisFilepath, finalDestFilename)
+		if err != nil {
+			clearFailedUploadChunks(uplaodedFiles)
+			return []string{}, err
+		}
+
+		uploadedFilepaths = append(uploadedFilepaths, finalDestFilename)
+		log.Println(userinfo.Username+" uploaded: ", filepath.Base(thisFilepath))
+	}
+
+	return uploadedFilepaths, nil
+}
+
+//This function remove all the chunks for failed file upload from disk
+func clearFailedUploadChunks(uplaodedFiles map[string]string) {
+	for _, tmpFiles := range uplaodedFiles {
+		os.Remove(tmpFiles)
+	}
+}

web/SystemAO/file_system/file_explorer.html

@@ -4281,8 +4281,8 @@
                 }
 
                 //Prase upload Form
-                let url = '../../system/file_system/upload'
                 let uploadCurrentPath = JSON.parse(JSON.stringify(currentPath));
+                let url = '../../system/file_system/upload?path=' + uploadCurrentPath
                 if (targetDir !== undefined){
                     //The upload paramter supplied targetDir
                     uploadCurrentPath = targetDir;