Added experimental upload system for handling huge file

file_system.go

@@ -429,15 +429,27 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//Generate an UUID for this upload
-	uploadUUID := uuid.NewV4().String()
-	uploadFolder := filepath.Join(*tmp_directory, "uploads", uploadUUID)
-	os.MkdirAll(uploadFolder, 0700)
+	//Check if it is huge file upload mode
+	isHugeFile := false
+	hugefile, _ := common.Mv(r, "hugefile", false)
+	if hugefile == "true" {
+		isHugeFile = true
+	}
+
 	targetUploadLocation := filepath.Join(realUploadPath, filename)
 	if !fs.FileExists(realUploadPath) {
 		os.MkdirAll(realUploadPath, 0755)
 	}
 
+	//Generate an UUID for this upload
+	uploadUUID := uuid.NewV4().String()
+	uploadFolder := filepath.Join(*tmp_directory, "uploads", uploadUUID)
+	if isHugeFile {
+		//Upload to the same directory as the target location.
+		uploadFolder = filepath.Join(realUploadPath, ".metadata/.upload", uploadUUID)
+	}
+	os.MkdirAll(uploadFolder, 0700)
+
 	//Start websocket connection
 	var upgrader = websocket.Upgrader{}
 	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
@@ -501,7 +513,22 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			//File block. Save it to tmp folder
 			chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(blockCounter))
 			chunkName = append(chunkName, chunkFilepath)
-			ioutil.WriteFile(chunkFilepath, message, 0700)
+			writeErr := ioutil.WriteFile(chunkFilepath, message, 0700)
+
+			if writeErr != nil {
+				//Unable to write block. Is the tmp folder fulled?
+				log.Println("[Upload] Upload chunk write failed: " + err.Error())
+				c.WriteMessage(1, []byte(`{\"error\":\"Write file chunk to disk failed\"}`))
+
+				//Close the connection
+				c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
+				time.Sleep(1 * time.Second)
+				c.Close()
+
+				//Clear the tmp files
+				os.RemoveAll(uploadFolder)
+				return
+			}
 
 			//Update the last upload chunk time
 			lastChunkArrivalTime = time.Now().Unix()
@@ -510,7 +537,7 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			totalFileSize += fs.GetFileSize(chunkFilepath)
 			if totalFileSize > max_upload_size {
 				//File too big
-				c.WriteMessage(1, []byte(`{\"error\":\"File size too large.\"}`))
+				c.WriteMessage(1, []byte(`{\"error\":\"File size too large\"}`))
 
 				//Close the connection
 				c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
@@ -571,6 +598,9 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 		}
 		io.Copy(out, srcChunkReader)
 		srcChunkReader.Close()
+
+		//Delete file immediately to save space
+		os.Remove(filesrc)
 	}
 
 	out.Close()

web/SystemAO/file_system/file_explorer.html

@@ -1176,6 +1176,7 @@
             let maxConcurrentUpload = 4; //Maxmium number of oncurrent upload
             let uploadPendingList = []; //Upload pending queue for mass upoad
             let lowMemoryMode = true;   //Upload with low memory mode channel
+            let largeFileCutoffSize = 8192 * 1024 * 1024; //Any file larger than this size is consider "large file", default to 8GB
             let uploadFileChunkSize = 1024 * 512; //512KB, 4MB not working quite well on slow network
 
             //File Sharing related
@@ -4608,6 +4609,8 @@
         }
 
         function initUploadMode(){
+            //Deprecated, will automatically decide which mode to use base on file size and disk space
+            /*
             //Get the amount of RAM on the server side to check if low memory mode should be used.
             $.ajax({
                 url: "../../system/info/getRAMinfo",
@@ -4616,7 +4619,7 @@
                         //Permission denied or other reasons that cannot access harwdare info. Use default mode
                         lowMemoryMode = true;
                     }else{
-                        //Check if ramsize > 1.8 GB (2GB). If yes, switch to large memory upload mode
+                        //Check if ramsize > 3.8GB. If yes, switch to large memory upload mode
                         var memsize = JSON.parse(data);
                         if (parseFloat(memsize)/ 1024 / 1024 / 1024 >= 3.8){
                             console.log("[File Explorer] Entering large memory upload mode")
@@ -4629,12 +4632,31 @@
                     lowMemoryMode = true;
                 }
             });
+            */
+
+            //Get the avaible space on tmp disk and decide the cutoff file size that need to directly write to disk
+            $.ajax({
+                url: "../../system/disk/space/resolve",
+                method: "POST",
+                data: {path: "tmp:/"},
+                success: function(data){
+                    if (data.error !== undefined){
+                      
+                    }else{
+                        console.log("[File Explorer] Setting huge file cutoff size at: " + ao_module_utils.formatBytes(data.Avilable/16));
+                        largeFileCutoffSize = data.Avilable/16 - 4096;
+                    }
+                },
+                error: function(){
+                    //Hardware mode disabled. Use default value.
+                }
+            });
         }
 
         let uploadBufferedRefreshTimer;
        
         function uploadFile(file, uuid=undefined, targetDir=undefined) {
-            if (lowMemoryMode){
+            if (file.size > 4194304 && lowMemoryMode){
                  /*
                     Low Memory Upload Mode
                 */
@@ -4720,7 +4742,13 @@
                     //console.log("Firefox Mode: ", uploadDir, "Target Dir", targetDir)
                 }
 
-                let socket = new WebSocket(protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + encodeURIComponent(filename) + "&path=" + encodeURIComponent(uploadDir));
+                let hugeFileMode = "";
+                if (file.size > largeFileCutoffSize){
+                    //Filesize over cutoff line. Use huge file mode
+                    hugeFileMode = "&hugefile=true";
+                }
+
+                let socket = new WebSocket(protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + encodeURIComponent(filename) + "&path=" + encodeURIComponent(uploadDir) + hugeFileMode);
                 let currentSendingIndex = 0;
                 let chunks = Math.ceil(file.size/uploadFileChunkSize,uploadFileChunkSize);
                 

web/SystemAO/file_system/zip_extractor.html

@@ -35,7 +35,7 @@
             var oprConfig = {
                 opr: "unzipAndOpen",
                 src: targetFilePaths,
-                dest: "tmp:/temp_" + Date.now() + "/",
+                dest: "tmp:/tmp_" + Date.now() + "/",
                 overwriteMode: "overwrite",
             }