auto update script executed

api.go

@@ -121,6 +121,8 @@ func initAPIs() {
 	authRouter.HandleFunc("/api/analytic/list", AnalyticLoader.HandleSummaryList)
 	authRouter.HandleFunc("/api/analytic/load", AnalyticLoader.HandleLoadTargetDaySummary)
 	authRouter.HandleFunc("/api/analytic/loadRange", AnalyticLoader.HandleLoadTargetRangeSummary)
+	authRouter.HandleFunc("/api/analytic/exportRange", AnalyticLoader.HandleRangeExport)
+	authRouter.HandleFunc("/api/analytic/resetRange", AnalyticLoader.HandleRangeReset)
 
 	//Network utilities
 	authRouter.HandleFunc("/api/tools/ipscan", HandleIpScan)

mod/statistic/analytic/analytic.go

@@ -1,10 +1,9 @@
 package analytic
 
 import (
-	"encoding/json"
+	"errors"
 	"net/http"
 	"strings"
-	"time"
 
 	"imuslab.com/zoraxy/mod/database"
 	"imuslab.com/zoraxy/mod/statistic"
@@ -24,105 +23,49 @@ func NewDataLoader(db *database.Database, sc *statistic.Collector) *DataLoader {
 	}
 }
 
-func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) {
-	entries, err := d.Database.ListTable("stats")
-	if err != nil {
-		utils.SendErrorResponse(w, "unable to load data from database")
-		return
-	}
-
-	entryDates := []string{}
-	for _, keypairs := range entries {
-		entryDates = append(entryDates, string(keypairs[0]))
-	}
-
-	js, _ := json.MarshalIndent(entryDates, "", " ")
-	utils.SendJSONResponse(w, string(js))
-}
-
-func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) {
-	day, err := utils.GetPara(r, "id")
+// GetAllStatisticSummaryInRange return all the statisics within the time frame. The second array is the key (dates) of the statistic
+func (d *DataLoader) GetAllStatisticSummaryInRange(start, end string) ([]*statistic.DailySummaryExport, []string, error) {
+	dailySummaries := []*statistic.DailySummaryExport{}
+	collectedDates := []string{}
+	//Generate all the dates in between the range
+	keys, err := generateDateRange(start, end)
 	if err != nil {
-		utils.SendErrorResponse(w, "id cannot be empty")
-		return
+		return dailySummaries, collectedDates, err
 	}
 
-	if strings.Contains(day, "-") {
-		//Must be underscore
-		day = strings.ReplaceAll(day, "-", "_")
-	}
-
-	if !statistic.IsBeforeToday(day) {
-		utils.SendErrorResponse(w, "given date is in the future")
-		return
-	}
-
-	var targetDailySummary statistic.DailySummaryExport
-
-	if day == time.Now().Format("2006_01_02") {
-		targetDailySummary = *d.StatisticCollector.GetExportSummary()
-	} else {
-		//Not today data
-		err = d.Database.Read("stats", day, &targetDailySummary)
-		if err != nil {
-			utils.SendErrorResponse(w, "target day data not found")
-			return
+	//Load all the data from database
+	for _, key := range keys {
+		thisStat := statistic.DailySummaryExport{}
+		err = d.Database.Read("stats", key, &thisStat)
+		if err == nil {
+			dailySummaries = append(dailySummaries, &thisStat)
+			collectedDates = append(collectedDates, key)
 		}
 	}
 
-	js, _ := json.Marshal(targetDailySummary)
-	utils.SendJSONResponse(w, string(js))
+	return dailySummaries, collectedDates, nil
+
 }
 
-func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) {
-	//Get the start date from POST para
+func (d *DataLoader) GetStartAndEndDatesFromRequest(r *http.Request) (string, string, error) {
+	// Get the start date from POST para
 	start, err := utils.GetPara(r, "start")
 	if err != nil {
-		utils.SendErrorResponse(w, "start date cannot be empty")
-		return
+		return "", "", errors.New("start date cannot be empty")
 	}
 	if strings.Contains(start, "-") {
 		//Must be underscore
 		start = strings.ReplaceAll(start, "-", "_")
 	}
-	//Get end date from POST para
+	// Get end date from POST para
 	end, err := utils.GetPara(r, "end")
 	if err != nil {
-		utils.SendErrorResponse(w, "emd date cannot be empty")
-		return
+		return "", "", errors.New("end date cannot be empty")
 	}
 	if strings.Contains(end, "-") {
 		//Must be underscore
 		end = strings.ReplaceAll(end, "-", "_")
 	}
 
-	//Generate all the dates in between the range
-	keys, err := generateDateRange(start, end)
-	if err != nil {
-		utils.SendErrorResponse(w, err.Error())
-		return
-	}
-
-	//Load all the data from database
-	dailySummaries := []*statistic.DailySummaryExport{}
-	for _, key := range keys {
-		thisStat := statistic.DailySummaryExport{}
-		err = d.Database.Read("stats", key, &thisStat)
-		if err == nil {
-			dailySummaries = append(dailySummaries, &thisStat)
-		}
-	}
-
-	//Merge the summaries into one
-	mergedSummary := mergeDailySummaryExports(dailySummaries)
-
-	js, _ := json.Marshal(struct {
-		Summary *statistic.DailySummaryExport
-		Records []*statistic.DailySummaryExport
-	}{
-		Summary: mergedSummary,
-		Records: dailySummaries,
-	})
-
-	utils.SendJSONResponse(w, string(js))
+	return start, end, nil
 }

mod/statistic/analytic/handlers.go

@@ -0,0 +1,218 @@
+package analytic
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"imuslab.com/zoraxy/mod/statistic"
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) {
+	entries, err := d.Database.ListTable("stats")
+	if err != nil {
+		utils.SendErrorResponse(w, "unable to load data from database")
+		return
+	}
+
+	entryDates := []string{}
+	for _, keypairs := range entries {
+		entryDates = append(entryDates, string(keypairs[0]))
+	}
+
+	js, _ := json.MarshalIndent(entryDates, "", " ")
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) {
+	day, err := utils.GetPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "id cannot be empty")
+		return
+	}
+
+	if strings.Contains(day, "-") {
+		//Must be underscore
+		day = strings.ReplaceAll(day, "-", "_")
+	}
+
+	if !statistic.IsBeforeToday(day) {
+		utils.SendErrorResponse(w, "given date is in the future")
+		return
+	}
+
+	var targetDailySummary statistic.DailySummaryExport
+
+	if day == time.Now().Format("2006_01_02") {
+		targetDailySummary = *d.StatisticCollector.GetExportSummary()
+	} else {
+		//Not today data
+		err = d.Database.Read("stats", day, &targetDailySummary)
+		if err != nil {
+			utils.SendErrorResponse(w, "target day data not found")
+			return
+		}
+	}
+
+	js, _ := json.Marshal(targetDailySummary)
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	dailySummaries, _, err := d.GetAllStatisticSummaryInRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Merge the summaries into one
+	mergedSummary := mergeDailySummaryExports(dailySummaries)
+
+	js, _ := json.Marshal(struct {
+		Summary *statistic.DailySummaryExport
+		Records []*statistic.DailySummaryExport
+	}{
+		Summary: mergedSummary,
+		Records: dailySummaries,
+	})
+
+	utils.SendJSONResponse(w, string(js))
+}
+
+// Handle exporting of a given range statistics
+func (d *DataLoader) HandleRangeExport(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	dailySummaries, dates, err := d.GetAllStatisticSummaryInRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	format, err := utils.GetPara(r, "format")
+	if err != nil {
+		format = "json"
+	}
+
+	if format == "csv" {
+		// Create a buffer to store CSV content
+		var csvContent strings.Builder
+
+		// Create a CSV writer
+		writer := csv.NewWriter(&csvContent)
+
+		// Write the header row
+		header := []string{"Date", "TotalRequest", "ErrorRequest", "ValidRequest", "ForwardTypes", "RequestOrigin", "RequestClientIp", "Referer", "UserAgent", "RequestURL"}
+		err := writer.Write(header)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Write each data row
+		for i, item := range dailySummaries {
+			row := []string{
+				dates[i],
+				strconv.FormatInt(item.TotalRequest, 10),
+				strconv.FormatInt(item.ErrorRequest, 10),
+				strconv.FormatInt(item.ValidRequest, 10),
+				// Convert map values to a comma-separated string
+				strings.Join(mapToStringSlice(item.ForwardTypes), ","),
+				strings.Join(mapToStringSlice(item.RequestOrigin), ","),
+				strings.Join(mapToStringSlice(item.RequestClientIp), ","),
+				strings.Join(mapToStringSlice(item.Referer), ","),
+				strings.Join(mapToStringSlice(item.UserAgent), ","),
+				strings.Join(mapToStringSlice(item.RequestURL), ","),
+			}
+			err = writer.Write(row)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		// Flush the CSV writer
+		writer.Flush()
+
+		// Check for any errors during writing
+		if err := writer.Error(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Set the response headers
+		w.Header().Set("Content-Type", "text/csv")
+		w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".csv")
+
+		// Write the CSV content to the response writer
+		_, err = w.Write([]byte(csvContent.String()))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	} else if format == "json" {
+		type exportData struct {
+			Stats []*statistic.DailySummaryExport
+			Dates []string
+		}
+
+		results := exportData{
+			Stats: dailySummaries,
+			Dates: dates,
+		}
+
+		js, _ := json.MarshalIndent(results, "", " ")
+		w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".json")
+		utils.SendJSONResponse(w, string(js))
+	} else {
+		utils.SendErrorResponse(w, "Unsupported export format")
+	}
+}
+
+// Reset all the keys within the given time period
+func (d *DataLoader) HandleRangeReset(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	if r.Method != http.MethodDelete {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	keys, err := generateDateRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	for _, key := range keys {
+		log.Println("DELETING statistics " + key)
+		d.Database.Delete("stats", key)
+
+		if isTodayDate(key) {
+			//It is today's date. Also reset statistic collector value
+			log.Println("RESETING today's in-memory statistics")
+			d.StatisticCollector.ResetSummaryOfDay()
+		}
+	}
+
+	utils.SendOK(w)
+}

mod/statistic/analytic/utils.go

@@ -70,3 +70,25 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 
 	return mergedExport
 }
+
+func mapToStringSlice(m map[string]int) []string {
+	slice := make([]string, 0, len(m))
+	for k := range m {
+		slice = append(slice, k)
+	}
+	return slice
+}
+
+func isTodayDate(dateStr string) bool {
+	today := time.Now().Local().Format("2006-01-02")
+	inputDate, err := time.Parse("2006-01-02", dateStr)
+	if err != nil {
+		inputDate, err = time.Parse("2006_01_02", dateStr)
+		if err != nil {
+			fmt.Println("Invalid date format")
+			return false
+		}
+	}
+
+	return inputDate.Format("2006-01-02") == today
+}

mod/statistic/statistic.go

@@ -97,6 +97,11 @@ func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *Daily
 	return &targetSummary
 }
 
+// Reset today summary, for debug or restoring injections
+func (c *Collector) ResetSummaryOfDay() {
+	c.DailySummary = newDailySummary()
+}
+
 // This function gives the current slot in the 288- 5 minutes interval of the day
 func (c *Collector) GetCurrentRealtimeStatIntervalId() int {
 	now := time.Now()

web/components/stats.html

@@ -16,7 +16,7 @@
             </div>
         </div>
         <button onclick="handleLoadStatisticButtonPress();" class="ui basic button"><i class="blue search icon"></i> Load</button>
-        <button onclick="clearStatisticDateRange();" class="ui basic button"><i class="eraser icon"></i> Clear Results</button>
+        <button onclick="clearStatisticDateRange();" class="ui basic button"><i class="eraser icon"></i> Clear Search</button>
         <br>
         <small>Leave end range as empty for showing starting day only statistic</small>
     </div>

web/snippet/advanceStatsOprs.html

@@ -16,13 +16,53 @@
                 </div>
             </div>
             <div class="ui divider"></div>
-            
+            <h3>Export Data</h3>
+            <p>You can export the statistics collected by Zoraxy in the selected range for further analysis</p>
+            <button class="ui basic teal button" onclick="handleExportAsCSV();"><i class="download icon"></i> Export CSV</button>
+            <button class="ui basic pink button" onclick="handleExportAsJSON();"><i class="download icon"></i> Export JSON</button>
+            <div class="ui divider"></div>
+            <h3>Reset Statistics</h3>
+            <p>You can reset the statistics within the selected time range for debug purpose. Note that this operation is irreversible.</p>
+            <button class="ui basic red button" onclick="handleResetStats();"><i class="trash icon"></i> RESET STATISTICS</button>
+            <br><br>
             <button class="ui basic button iframeOnly"  style="float: right;" onclick="parent.hideSideWrapper();"><i class="remove icon"></i> Cancel</button>
         </div>
         <script>
             let startDate = "";
             let endDate = "";
 
+            /*
+                Actions Handler
+            */
+
+            function handleExportAsJSON(){
+                window.open(`/api/analytic/exportRange?start=${startDate}&end=${endDate}&format=json`, 'download');
+            }
+
+            function handleExportAsCSV(){
+                window.open(`/api/analytic/exportRange?start=${startDate}&end=${endDate}&format=csv`, 'download');
+            }
+
+            function handleResetStats(){
+                if (confirm("Confirm remove statistics from " + startDate + " to " + endDate +"?")){
+                    $.ajax({
+                        url: "/api/analytic/resetRange?start=" + startDate + "&end=" + endDate,
+                        method: "DELETE",
+                        success: function(data){
+                            if (data.error != undefined){
+                                parent.msgbox(data.error, false, 5000);
+                            }else{
+                                parent.msgbox("Statistic Cleared");
+                                parent.hideSideWrapper();
+                            }
+                        }
+                    })
+                }
+            }
+
+            /*
+                Data Loading
+            */
             function loadDateRange(){
                 if (window.location.hash.length > 1){
                     try{