auto update script executed

mod/dynamicproxy/proxyRequestHandler.go

@@ -149,6 +149,9 @@ func (h *ProxyHandler) logRequest(r *http.Request, succ bool, statusCode int, fo
 				Succ:                          succ,
 				StatusCode:                    statusCode,
 				ForwardType:                   forwardType,
+				Referer:                       r.Referer(),
+				UserAgent:                     r.UserAgent(),
+				RequestURL:                    r.RequestURI,
 			}
 			h.Parent.Option.StatisticCollector.RecordRequest(requestInfo)
 		}()

mod/statistic/statistic.go

@@ -1,6 +1,7 @@
 package statistic
 
 import (
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -25,6 +26,9 @@ type DailySummary struct {
 	ForwardTypes    *sync.Map //Map that hold the forward types
 	RequestOrigin   *sync.Map //Map that hold [country ISO code]: visitor counter
 	RequestClientIp *sync.Map //Map that hold all unique request IPs
+	Referer         *sync.Map //Map that store where the user was refered from
+	UserAgent       *sync.Map //Map that store the useragent of the request
+	RequestURL      *sync.Map //Request URL of the request object
 }
 
 type RequestInfo struct {
@@ -33,6 +37,9 @@ type RequestInfo struct {
 	Succ                          bool
 	StatusCode                    int
 	ForwardType                   string
+	Referer                       string
+	UserAgent                     string
+	RequestURL                    string
 }
 
 type CollectorOption struct {
@@ -148,6 +155,38 @@ func (c *Collector) RecordRequest(ri RequestInfo) {
 		} else {
 			c.DailySummary.RequestClientIp.Store(ri.IpAddr, fi.(int)+1)
 		}
+
+		//Record the referer
+		rf, ok := c.DailySummary.Referer.Load(ri.Referer)
+		if !ok {
+			c.DailySummary.Referer.Store(ri.Referer, 1)
+		} else {
+			c.DailySummary.Referer.Store(ri.Referer, rf.(int)+1)
+		}
+
+		//Record the UserAgent
+		ua, ok := c.DailySummary.UserAgent.Load(ri.UserAgent)
+		if !ok {
+			c.DailySummary.UserAgent.Store(ri.UserAgent, 1)
+		} else {
+			c.DailySummary.UserAgent.Store(ri.UserAgent, ua.(int)+1)
+		}
+
+		//ADD MORE HERE IF NEEDED
+
+		//Record request URL, if it is a page
+		ext := filepath.Ext(ri.RequestURL)
+
+		if ext != "" && !isWebPageExtension(ext) {
+			return
+		}
+
+		ru, ok := c.DailySummary.RequestURL.Load(ri.RequestURL)
+		if !ok {
+			c.DailySummary.RequestURL.Store(ri.RequestURL, 1)
+		} else {
+			c.DailySummary.RequestURL.Store(ri.RequestURL, ru.(int)+1)
+		}
 	}()
 }
 
@@ -190,5 +229,8 @@ func newDailySummary() *DailySummary {
 		ForwardTypes:    &sync.Map{},
 		RequestOrigin:   &sync.Map{},
 		RequestClientIp: &sync.Map{},
+		Referer:         &sync.Map{},
+		UserAgent:       &sync.Map{},
+		RequestURL:      &sync.Map{},
 	}
 }

mod/statistic/structconv.go

@@ -10,6 +10,9 @@ type DailySummaryExport struct {
 	ForwardTypes    map[string]int
 	RequestOrigin   map[string]int
 	RequestClientIp map[string]int
+	Referer         map[string]int
+	UserAgent       map[string]int
+	RequestURL      map[string]int
 }
 
 func DailySummaryToExport(summary DailySummary) DailySummaryExport {
@@ -20,6 +23,9 @@ func DailySummaryToExport(summary DailySummary) DailySummaryExport {
 		ForwardTypes:    make(map[string]int),
 		RequestOrigin:   make(map[string]int),
 		RequestClientIp: make(map[string]int),
+		Referer:         make(map[string]int),
+		UserAgent:       make(map[string]int),
+		RequestURL:      make(map[string]int),
 	}
 
 	summary.ForwardTypes.Range(func(key, value interface{}) bool {
@@ -37,6 +43,21 @@ func DailySummaryToExport(summary DailySummary) DailySummaryExport {
 		return true
 	})
 
+	summary.Referer.Range(func(key, value interface{}) bool {
+		export.Referer[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.UserAgent.Range(func(key, value interface{}) bool {
+		export.UserAgent[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestURL.Range(func(key, value interface{}) bool {
+		export.RequestURL[key.(string)] = value.(int)
+		return true
+	})
+
 	return export
 }
 
@@ -48,6 +69,9 @@ func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
 		ForwardTypes:    &sync.Map{},
 		RequestOrigin:   &sync.Map{},
 		RequestClientIp: &sync.Map{},
+		Referer:         &sync.Map{},
+		UserAgent:       &sync.Map{},
+		RequestURL:      &sync.Map{},
 	}
 
 	for k, v := range export.ForwardTypes {
@@ -62,5 +86,17 @@ func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
 		summary.RequestClientIp.Store(k, v)
 	}
 
+	for k, v := range export.Referer {
+		summary.Referer.Store(k, v)
+	}
+
+	for k, v := range export.UserAgent {
+		summary.UserAgent.Store(k, v)
+	}
+
+	for k, v := range export.RequestURL {
+		summary.RequestURL.Store(k, v)
+	}
+
 	return summary
 }

mod/statistic/utils.go

@@ -0,0 +1,11 @@
+package statistic
+
+func isWebPageExtension(ext string) bool {
+	webPageExts := []string{".html", ".htm", ".php", ".jsp", ".aspx", ".js", ".jsx"}
+	for _, e := range webPageExts {
+		if e == ext {
+			return true
+		}
+	}
+	return false
+}

tools/start.bat

@@ -0,0 +1 @@
+.\boltbrowser.win64.exe ../sys.db