Zoraxy/web/components/stats.html

<script src="./script/useragent.js"></script>
<link href="./script/datepicker/dp-light.css" rel="stylesheet">
<script defer src="./script/datepicker/datepicker.js"></script>

<div class="standardContainer">
    <div class="ui basic segment">
        <h2>Statistical Analysis</h2>
        <p>Statistic of your server in every aspects</p>
        <div style="margin-bottom: 0.4em;">
            <div class="ui small input">
                <input type="text" id="statsRangeStart" placeholder="From date">
            </div>
            <span style="padding-left: 0.6em; padding-right: 0.6em;"> <i class="ui right arrow icon"></i> </span>
            <div class="ui small input">
                <input type="text" id="statsRangeEnd" placeholder="End date">
            </div>
        </div>
        <button onclick="handleLoadStatisticButtonPress();" class="ui basic button"><i class="blue search icon"></i> Search</button>
        <button onclick="clearStatisticDateRange();" class="ui yellow basic button"><i class="eraser icon"></i> Clear Range</button>
        <br>
        <small>Leave end range as empty for showing starting day only statistic</small>
    </div>
    <div class="ui divider"></div>
    <div id="statisticRenderNotEnoughData" class="ui segment" style="padding: 2em;" align="center">
        <h4 class="ui icon header">
            <i class="medium icons" style="color: #dbdbdb !important;">
                <i class="chart pie icon"></i>
                <i class="small corner remove icon" style="
                    font-size: 1.4em;
                    margin-right: -0.2em;
                    margin-bottom: -0em;
                "></i>
            </i>
            
            <div class="content" style="margin-top: 1em; color: #7c7c7c !important;">
                No Data 
                <div class="sub header" style="color: #adadad !important;">The selected period contains too little or no request data collected. <br>
                    Please select a larger range or make sure there are enough traffic routing through this site.</div>
            </div>
        </h4>
    </div>
    <div id="statisticRenderElement" class="ui basic segment">
        <!-- View Counts Statistics -->
        <div class="ui three small statistics">
            <div class="statistic">
              <div class="value totalViewCount">
                
              </div>
              <div class="label">
                Total Requests
              </div>
            </div>
            <div class="statistic">
                <div class="value">
                    <i class="ui green check circle icon"></i> <span class="totalSuccCount"></span>
                </div>
              <div class="label">
                 Success Requests
              </div>
            </div>
            <div class="statistic">
              <div class="value">
                <i class="ui red times circle icon"></i> <span class="totalErrorCount"></span>
              </div>
              <div class="label">
                Error Requests
              </div>
            </div>
        </div>
        <!-- Forward Type Data -->
        <h3>Forward Traffic Types</h3>
        <p>Traffic forwarding type classified by their protocols and proxy rules.</p>
        <table class="ui celled unstackable table">
            <thead>
                <tr><th>Forward Type</th>
                <th>Counts</th>
                <th>Percentage</th>
            </tr></thead>
            <tbody class="forwardTypeCounts">
            </tbody>
        </table>
        <!-- Client Geolocation Analysis-->
        <h3>Visitors Countries</h3>
        <p>Distributions of visitors by country code. Access origin are estimated using open source GeoIP database and might not be accurate.</p>
        <div style="min-height: 400px;">
            <canvas id="stats_visitors"></canvas>
        </div>
        <div class="ui divider"></div>
        <!-- Client IP Analysis -->
        <div class="ui stackable grid">
            <div class="eight wide column">
                <h3>Requests IP Version</h3>
                <p>The version of Internet Protocol that the client is using. If the request client is pass through a DNS proxy like CloudFlare, 
                    the CloudFlare proxy server address will be filtered and only the first value in the RemoteAddress field will be analysised.</p>
                <div>
                    <canvas id="stats_ipver"></canvas>
                </div>
            </div>
            <div class="eight wide column">
                <h3>Request Origins</h3>
                <p>Top 25 request origin sorted by request count</p>
                <div style="height: 500px; overflow-y: auto;">
                    <table class="ui unstackable striped celled table">
                        <thead>
                          <tr>
                            <th class="no-sort">Request Origin</th>
                            <th class="no-sort">No of Requests</th>
                        </tr></thead>
                        <tbody id="stats_requestCountlist">
                         
                        </tbody>
                    </table>
                </div>
            </div>
        </div>
        <div class="ui divider"></div>
        <!-- Client Device Analysis -->
        <div class="ui stackable grid">
            <div class="eight wide column">
                <h3>Client Devices</h3>
                <p>Device type analysis by its request interactions.The number of iteration count does not means the number unique device, as no cookie is used to track the devices identify.</p>
                <div>
                    <canvas id="stats_device"></canvas>
                </div>
            </div>
            <div class="eight wide column">
                <h3>Client Browsers</h3>
                <p>The browsers where your client is using to visit your site</p>
                <div>
                    <canvas id="stats_browsers"></canvas>
                </div>
            </div>
        </div>
        <div class="ui stackable grid">
            <div class="eight wide column">
                <h3>Client OS</h3>
                <p>The OS where your client is using. Estimated using the UserAgent header sent by the client browser while requesting a resources from one of your host.</p>
                <div>
                    <canvas id="stats_OS"></canvas>
                </div>
            </div>
            <div class="eight wide column">
                <h3>OS Versions</h3>
                <p>The OS versions most commonly used by your site's visitors.</p>
                <div style="height: 500px; overflow-y: auto;">
                    <table class="ui unstackable striped celled table">
                        <thead>
                          <tr>
                            <th>OS Version</th>
                            <th>Request Counts</th>
                            <th>Percentage</th>
                        </tr></thead>
                        <tbody id="stats_OSVersionList">
                         
                        </tbody>
                    </table>
                </div>
            </div>
        </div>
        <div class="ui divider"></div>
        <div class="ui stackable grid">
            <div class="eight wide column">
                <h3>Request File Types</h3>
                <p>The file types being served by this proxy</p>
                <div>
                    <canvas id="stats_filetype"></canvas>
                </div>
            </div>
            <div class="eight wide column">
                <h3>Referring Sites</h3>
                <p>The Top 100 sources of traffic according to referer header</p>
                <div>
                    <div style="height: 500px; overflow-y: auto;">
                        <table class="ui unstackable striped celled table">
                            <thead>
                              <tr>
                                <th class="no-sort">Referer</th>
                                <th class="no-sort">Requests</th>
                            </tr></thead>
                            <tbody id="stats_RefererTable">
                             
                            </tbody>
                        </table>
                    </div>
                </div>
            </div>
        </div>  
        <div class="ui divider"></div>
        <div class="ui basic segment" id="trendGraphs">
            <h3>Visitor Trend Analysis</h3>
            <p>Request trends in the selected time range</p>
            <div>
                <canvas id="requestTrends"></canvas>
            </div>
        </div>
    </div>
    <!-- <button class="ui icon right floated basic button" onclick="initStatisticSummery();"><i class="green refresh icon"></i> Refresh</button> -->
    <br><br>
</div>
<script>
    var statisticCharts = [];

    //Start day must be earlier than end date
    function initStatisticSummery(startdate="", enddate=""){
            if (startdate == "" && enddate == "" ){
                let todaykey = getTodayStatisticKey();
                loadStatisticByDate(todaykey);
            }else if ((startdate != "" && enddate == "") || startdate == enddate){
                //Load the target date
                let targetDate = startdate.trim();
                loadStatisticByDate(targetDate);
            }else{
                //Two dates are given and they are not identical
                loadStatisticByRange(startdate, enddate);
                //console.log(startdate, enddate);
            }
    }

   function loadStatisticByDate(dateid){
        $.getJSON("/api/analytic/load?id=" + dateid, function(data){
            if (data.error != undefined){
                msgbox(data.error, false, 6000);
                return;
            }
            
            //Destroy all the previous charts
            statisticCharts.forEach(function(thisChart){
                thisChart.destroy();
            })

            if (data.TotalRequest == 0){
                //No data to analysis
                $("#statisticRenderElement").hide()
                $("#statisticRenderNotEnoughData").show();
                return;
            }else{
                $("#statisticRenderElement").show();
                $("#statisticRenderNotEnoughData").hide();
            }

            //Render the text values
            $("#statisticRenderElement").find(".totalViewCount").text(abbreviateNumber(data.TotalRequest));
            $("#statisticRenderElement").find(".totalSuccCount").text(abbreviateNumber(data.ValidRequest));
            $("#statisticRenderElement").find(".totalErrorCount").text(abbreviateNumber(data.ErrorRequest));

            //Render forward type data
            renderForwardTypeCounts(data.ForwardTypes);
            
            //Render visitor data
            renderVisitorChart(data.RequestOrigin);

            //Render IP versions
            renderIPVersionChart(data.RequestClientIp);

            //Render user agent analysis
            renderUserAgentCharts(data.UserAgent);

            //Render file type by analysising request URL paths
            renderFileTypeGraph(data.RequestURL);

            //Render Referer header
            renderRefererTable(data.Referer);

            //Hide the trend graphs
            $("#trendGraphs").hide();
        });
   }
   initStatisticSummery();
   $("#statsRangeStart").val(getTodayStatisticKey().split("_").join("-"));

   function loadStatisticByRange(startdate, endDate){
        $.getJSON("/api/analytic/loadRange?start=" + startdate + "&end=" + endDate, function(data){
            //console.log(data);
            //Destroy all the previous charts
           
            statisticCharts.forEach(function(thisChart){
                thisChart.destroy();
            })
 
            if (data.Summary.TotalRequest == 0){
                //No data to analysis
                $("#statisticRenderElement").hide()
                $("#statisticRenderNotEnoughData").show();
                return;
            }else{
                $("#statisticRenderElement").show();
                $("#statisticRenderNotEnoughData").hide();
            }

            //Render the text values
            $("#statisticRenderElement").find(".totalViewCount").text(abbreviateNumber(data.Summary.TotalRequest));
            $("#statisticRenderElement").find(".totalSuccCount").text(abbreviateNumber(data.Summary.ValidRequest));
            $("#statisticRenderElement").find(".totalErrorCount").text(abbreviateNumber(data.Summary.ErrorRequest));

            //Render forward type data
            renderForwardTypeCounts(data.Summary.ForwardTypes);

            //Render visitor data
            renderVisitorChart(data.Summary.RequestOrigin);

            //Render IP versions
            renderIPVersionChart(data.Summary.RequestClientIp);

            //Render user agent analysis
            renderUserAgentCharts(data.Summary.UserAgent);

            //Render file type by analysising request URL paths
            renderFileTypeGraph(data.Summary.RequestURL);
            
            //Render Referer header
            renderRefererTable(data.Summary.Referer);

            //Render the trend graph
            $("#trendGraphs").show();
            renderTrendGraph(data.Records);
        });
   }

   picker.attach({ target: document.getElementById("statsRangeStart") });
   picker.attach({ target: document.getElementById("statsRangeEnd") });

   function renderForwardTypeCounts(forwardTypes){
        let tablBody = $("#statisticRenderElement").find(".forwardTypeCounts");
        tablBody.empty();
        let totalForwardCounts = 0;
        for (let [key, value] of Object.entries(forwardTypes)) {
            totalForwardCounts += value;
        }

        for (let [key, value] of Object.entries(forwardTypes)) {
            tablBody.append(`<tr>
                    <td>${key}</td>
                    <td>${abbreviateNumber(value)} (${value})</td>
                    <td>${((value/totalForwardCounts)*100).toFixed(3)}%</td>
                </tr>
            `);
        }
   }


   function getTodayStatisticKey(){
        var today = new Date();
        var year = today.getFullYear();
        var month = String(today.getMonth() + 1).padStart(2, '0');
        var day = String(today.getDate()).padStart(2, '0');

        var formattedDate = year + '_' + month + '_' + day;
        return formattedDate
   }

   function handleLoadStatisticButtonPress(){
        var sd = $("#statsRangeStart").val();
        var ed = $("#statsRangeEnd").val();
        //Swap them if sd is later than ed
        if (sd != "" && ed != "" && sd > ed) {
            ed = [sd, sd = ed][0];
            $("#statsRangeStart").val(sd);
            $("#statsRangeEnd").val(ed);
        }
       
        initStatisticSummery(sd, ed);
   }

   function clearStatisticDateRange(){
        $("#statsRangeStart").val("");
        $("#statsRangeEnd").val("");
   }

   function renderRefererTable(refererList){
        const sortedEntries = Object.entries(refererList).sort(([, valueA], [, valueB]) => valueB - valueA);
        $("#stats_RefererTable").html("");
        let endStop = 100;
        if (sortedEntries.length < 100){
            endStop = sortedEntries.length;
        }
        for (var i = 0; i < endStop; i++) {
            let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
            if (sortedEntries[i][0] == ""){
                //Root
                referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
            }
            $("#stats_RefererTable").append(`<tr>
                    <td>${referer}</td>
                    <td>${sortedEntries[i][1]}</td>
                </tr>`);
        }
   }

   function renderFileTypeGraph(requestURLs){
     //Create the device chart
     let fileExtensions = {};
     for (const [url, count] of Object.entries(requestURLs)) {
        let filename = url.split("/").pop();
        let ext = "";
        if (filename == ""){
            //Loading from a folder
            ext = "Folder path"
        }else{
            if (filename.includes(".")){
                ext = filename.split(".").pop();
            }else{
                ext = "API call"
            }
        }

        if (fileExtensions[ext] != undefined){
            fileExtensions[ext] = fileExtensions[ext] + count;
        }else{
            //First time this ext show up
            fileExtensions[ext] = count;
        }
     }

     //Convert the key-value pairs to array for graph render
     let fileTypes = [];
     let fileCounts = [];
     let colors = [];
     for (const [ftype, count] of Object.entries(fileExtensions)) {
        fileTypes.push(ftype);
        fileCounts.push(count);
        colors.push(generateColorFromHash(ftype));
     }  

     let filetypeChart = new Chart(document.getElementById("stats_filetype"), {
            type: 'pie',
            data: {
                labels: fileTypes,
                datasets: [{
                    data: fileCounts,
                    backgroundColor: colors,
                    hoverBackgroundColor: colors,
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
            }
        });

        statisticCharts.push(filetypeChart);
   }
   
   function renderTrendGraph(dailySummary){
        // Get the canvas element
        const canvas = document.getElementById('requestTrends');

        //Generate the X axis labels
        let datesLabel = [];
        let succData = [];
        let errorData = [];
        let totalData = [];
        for (var i = 0; i < dailySummary.length; i++){
            let thisDayData = dailySummary[i];
            datesLabel.push("Day " + i);
            succData.push(thisDayData.ValidRequest);
            errorData.push(thisDayData.ErrorRequest);
            totalData.push(thisDayData.TotalRequest);
        }
        // Create the chart
        let TrendChart = new Chart(canvas, {
            type: 'line',
            data: {
                labels: datesLabel,
                datasets: [
                    {
                        label: 'All Requests',
                        data: totalData,
                        borderColor: '#7d99f7',
                        backgroundColor: 'rgba(125, 153, 247, 0.4)',
                        fill: false
                    },
                    {
                        label: 'Success Requests',
                        data: succData,
                        borderColor: '#6dad7c',
                        backgroundColor: "rgba(109, 173, 124, 0.4)",
                        fill: true
                    },
                    {
                        label: 'Error Requests',
                        data: errorData,
                        borderColor: '#de7373',
                        backgroundColor: "rgba(222, 115, 115, 0.4)",
                        fill: true
                    },
                ]
            },
            options: {
                responsive: true,
                maintainAspectRatio: true,
                title: {
                    display: true,
                    //text: 'Line Chart Example'
                },
                scales: {
                    x: {
                        display: true,
                        title: {
                            display: false,
                            text: 'Time'
                        }
                    },
                    y: {
                        display: true,
                        title: {
                            display: true,
                            text: 'Request Counts'
                        }
                    }
                }
            }
        });

        statisticCharts.push(TrendChart);
   }

    function renderUserAgentCharts(userAgentsEntries){
        let userAgents = Object.keys(userAgentsEntries);
        let requestCounts = Object.values(userAgentsEntries);
        let mobileUser = 0;
        let desktopUser = 0;
        let osTypes = {};
        let osVersion = {};
        let browserTypes = {};
        let totalRequestCounts = 0;
        
        requestCounts.forEach(function(rc){
            totalRequestCounts += rc;
        })

        //Building a statistic summary
        userAgents.forEach(function(thisUA){
           var uaInfo = parseUserAgent(thisUA);
           if (uaInfo.isMobile){
                mobileUser+=userAgentsEntries[thisUA];
           }else{
                desktopUser+=userAgentsEntries[thisUA];
           }

           let currentNo = osTypes[uaInfo.os];
           let osVersionKey = uaInfo.os + " " + uaInfo.version.split("_").join(".");
           if (currentNo == undefined){
                osTypes[uaInfo.os] = userAgentsEntries[thisUA];
           }else{
                osTypes[uaInfo.os] = currentNo + userAgentsEntries[thisUA];
           }

           let p = osVersion[osVersionKey];
           if (p == undefined){
                osVersion[osVersionKey] = userAgentsEntries[thisUA];
           }else{
                osVersion[osVersionKey] = p + userAgentsEntries[thisUA];
           }


           let browserTypeKey = uaInfo.browser;
           if (browserTypeKey.indexOf("//") >= 0){
                //This is a uncatergorize browser, mostly a bot
                browserTypeKey = "Bots";
           }else if (browserTypeKey == ""){
                //No information
                browserTypeKey = "Unknown";
           }
           let b = browserTypes[browserTypeKey];
           if (b == undefined){
                browserTypes[browserTypeKey] = userAgentsEntries[thisUA];
           }else{
                browserTypes[browserTypeKey] = b + userAgentsEntries[thisUA];
           }
 
        });

        //Create the device chart
        let deviceTypeChart = new Chart(document.getElementById("stats_device"), {
            type: 'pie',
            data: {
                labels: ['Desktop', 'Mobile'],
                datasets: [{
                    data: [desktopUser, mobileUser],
                    backgroundColor: ['#e56b5e', '#6eb9c1'],
                    hoverBackgroundColor: ['#e56b5e', '#6eb9c1']
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
            }
        });

        statisticCharts.push(deviceTypeChart);

        //Create the OS chart
        let OSnames = [];
        let OSCounts = [];
        let OSColors = [];
        for (const [key, value] of Object.entries(osTypes)) {
            OSnames.push(key);
            OSCounts.push(value);
            OSColors.push(getOSColorCode(key));
        }
    
        let osTypeChart = new Chart(document.getElementById("stats_OS"), {
            type: 'pie',
            data: {
                labels: OSnames,
                datasets: [{
                    data: OSCounts,
                    backgroundColor: OSColors,
                    hoverBackgroundColor: OSColors
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
            }
        });

        statisticCharts.push(osTypeChart);

        //Populate the OS version table
        let sortedOSVersion = Object.entries(osVersion).sort((a, b) => b[1] - a[1]);
        $("#stats_OSVersionList").html("");
        // Loop through the sorted data and populate the table
        for (let i = 0; i < sortedOSVersion.length; i++) {
            let osVersion = sortedOSVersion[i][0];
            let requestcount = abbreviateNumber(sortedOSVersion[i][1]);
            let percentage = (sortedOSVersion[i][1] / totalRequestCounts * 100).toFixed(3);
            
            // Create a new row in the table
            let row = $("<tr>");
            
            // Add the OS version and percentage as columns in the row
            $("<td>").text(osVersion).appendTo(row);
            $("<td>").text(requestcount).appendTo(row);
            $("<td>").text(percentage + "%").appendTo(row);
            
            // Add the row to the table body
            $("#stats_OSVersionList").append(row);
        }

        //Create the browser charts
        let browserNames = [];
        let broserCounts = [];
        let browserColors = [];
        let sortedBrowserTypes = Object.entries(browserTypes).sort((a, b) => b[1] - a[1]);
        console.log(sortedBrowserTypes);
        sortedBrowserTypes.forEach(function(entry){
            browserNames.push(entry[0]);
            broserCounts.push(entry[1]);
            browserColors.push(generateColorFromHash(entry[0]));
        });
    
        let browserTypeChart = new Chart(document.getElementById("stats_browsers"), {
            type: 'pie',
            data: {
                labels: browserNames,
                datasets: [{
                    data: broserCounts,
                    backgroundColor: browserColors,
                    hoverBackgroundColor: browserColors
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
            }
        });

        statisticCharts.push(browserTypeChart);

    }

    //Generate the IPversion pie chart
    function renderIPVersionChart(RequestClientIp){
        let ipv4Count = Object.keys(RequestClientIp).filter(ip => ip.includes('.')).length;
        let ipv6Count = Object.keys(RequestClientIp).filter(ip => ip.includes(':')).length;
        let totalCount = ipv4Count + ipv6Count;
        let ipv4Percent = ((ipv4Count / totalCount) * 100).toFixed(2);
        let ipv6Percent = ((ipv6Count / totalCount) * 100).toFixed(2);

        // Create the chart data object
        let chartData = {
            labels: ['IPv4', 'IPv6'],
            datasets: [{
                data: [ipv4Percent, ipv6Percent],
                backgroundColor: ['#9295f0', '#bff092'],
                hoverBackgroundColor: ['#9295f0', '#bff092']
            }]
        };

        // Create the chart options object
        let ipvChartOption = {
            responsive: true,
            maintainAspectRatio: false,
        };

        // Create the pie chart
        let ipTypeChart = new Chart(document.getElementById("stats_ipver"), {
            type: 'pie',
            data: chartData,
            options: ipvChartOption
        });

        statisticCharts.push(ipTypeChart);

        //Populate the request count table
        let requestCounts = Object.entries(RequestClientIp);

        // Sort the array by the value (count)
        requestCounts.sort((a, b) => b[1] - a[1]);

        // Select the table body and empty it
        let tableBody = $('#stats_requestCountlist');
        tableBody.empty();

        // Loop through the sorted array and add the top 25 requested IPs to the table
        for (let i = 0; i < 25 && i < requestCounts.length; i++) {
            let [ip, count] = requestCounts[i];
            let row = $('<tr>').appendTo(tableBody);
            $('<td style="word-break: break-all;">').text(ip).appendTo(row);
            $('<td>').text(count).appendTo(row);
        }
    }

    //Generate a fixed color code from string hash
    function generateColorFromHash(stringToHash){
        let hash = 0;
        for (let i = 0; i < stringToHash.length; i++) {
            hash = stringToHash.charCodeAt(i) + ((hash << 5) - hash);
        }
        const hue = hash % 300;
        const saturation = Math.floor(Math.random() * 20) + 70;
        const lightness = Math.floor(Math.random() * 20) + 60;
        return `hsl(${hue}, ${saturation}%, ${lightness}%)`;
    }

    //Generate the visitor country pie chart    
   function renderVisitorChart(visitorData){
        // Extract the labels and data from the visitor data object
        let labels = [];
        let data = Object.values(visitorData);

        Object.keys(visitorData).forEach(function(cc){
            if (cc == ""){
                labels.push("Local / Unknown")
            }else{
                labels.push(`${getCountryName(cc)} [${cc.toUpperCase()}]` );
            }
        });

        // Define the colors to be used in the pie chart
        let colors = [];
        labels.forEach(function(cc){
            colors.push(generateColorFromHash(cc));
        });

        // Create the chart data object
        let CCchartData = {
            labels: labels,
            datasets: [{
                data: data,
                backgroundColor: colors
            }]
        };

        // Create the chart options object
        let CCchartOptions = {
            responsive: true,
            maintainAspectRatio: false,
        };

        // Create the pie chart
        const visitorsChart = new Chart(document.getElementById("stats_visitors"), {
            type: 'pie',
            data: CCchartData,
            options: CCchartOptions
        });

        statisticCharts.push(visitorsChart);
   }
</script>