[customized project] [M13] [data monitoring + data analysis + large visual screen] - Key Technology Python Flash + pandas + ecarts + Excel

Project requirements: monitor the changes of production log files in the specified directory (including subdirectories) of the disk, analyze the statistical data of the file (Excel format) and display it on the visualization screen.

Project duration: 5 working days.

2, Project architecture

The operating environment supports Windows and Linux systems.
B/S architecture, support Chrome, IE, QQ and other mainstream browsers.
http server: Python Flask framework;
File monitoring: Python win32con module;
Data analysis: Python Pandas library;
Data visualization: echorts + bootstrap library;

3, Project source code analysis

1. http server: Python Flask framework;

http code

from flask import Flask

#Other codes are omitted here
#...

@app.route('/datas/<flag>', methods=['GET', 'POST'])
def datas(flag):
    data = get_monitor(flag).get_result()
    return {
        "id": flag,
        "msg": "success",
        "data": data
    }

if __name__ == '__main__':
    # Open flash service
    app.run(host='0.0.0.0', port=8080, debug=True)

Interface resolution

Front end calling interface: Data

Parameter flag: monitored directory path

Data request method: GET or POST

Data response: JSON

2. File monitoring: Python win32con module;

Monitoring code

hDir = win32file.CreateFile(
            self.path_to_watch,
            FILE_LIST_DIRECTORY,
            win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
            None,
            win32con.OPEN_EXISTING,
            win32con.FILE_FLAG_BACKUP_SEMANTICS,
            None
        )
while 1:
    results = win32file.ReadDirectoryChangesW(
                hDir,
                1024,
                True,
                win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
                win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
                win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
                win32con.FILE_NOTIFY_CHANGE_SIZE |
                win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
                win32con.FILE_NOTIFY_CHANGE_SECURITY,
                None,
                None)
    for action, filename in results:
        full_filename = os.path.join(self.path_to_watch, filename)
        # Respond to event processing and start analysis and statistics
        # . . .

Code parsing

win32file.CreateFile monitoring directory path_to_watch.

win32file.ReadDirectoryChangesW responds to directory and subdirectory file change notifications.

3. Data analysis: Python Pandas library;

Pandas Series code

# Count the total number of keyword occurrences

df = pd.read_csv(file, encoding="gb2312")
return df['Survey item name'].value_counts()

# Merge data
series_total = series_total + series_config

# Delete nan
series_total = series_total.dropna()

File example

4. Data visualization: echorts + bootstrap library;

BootStrap layout

<div class="container_fluid">
        <!-- Title Block -->
        <div class="row_fluid">
            <div id="container_1" class="col-xs-12 col-md-12">
 
            </div>
        </div>
 
        <!-- Upper column -->
        <div class="row_fluid">
 
            <!-- Upper left column -->
            <div id="container_2" class="col-xs-12 col-md-4">
                <div id="container_2_1" class="col-xs-12 col-md-6">
                </div>
                <div id="container_2_2" class="col-xs-12 col-md-6">
                </div>
                <div id="container_2_3" class="col-xs-12 col-md-12">
                </div>
            </div>
 
            <!-- Upper and middle column -->
            <div id="container_3" class="col-xs-12 col-md-4">
                <iframe src="myimg/video.mp4" scrolling="no" border=0 frameborder="no" framespacing=0
                    allowfullscreen="true" width="100%" height="100%"> </iframe>
            </div>
 
            <!-- Upper right column -->
            <div id="container_4" class="col-xs-12 col-md-4">
                <div id="container_4_1" class="col-xs-12 col-md-3">
                </div>
 
                <div id="container_4_2" class="col-xs-12 col-md-3">
                </div>
 
                <div id="container_4_3" class="col-xs-12 col-md-3">
                </div>
 
                <div id="container_4_4" class="col-xs-12 col-md-3">
                </div>
 
                <div id="container_4_5" class="col-xs-12 col-md-6">
                </div>
 
                <div id="container_4_6" class="col-xs-12 col-md-6">
                </div>
            </div>
        </div>
    </div>
 
    <!-- Lower column -->
    <div class="row_fluid">
        <!-- Lower left column -->
        <div id="container_5" class="col-xs-12 col-md-4">
 
            <div id="container_5_1" class="col-xs-12 col-md-12">
            </div>
            <div id="container_5_2" class="col-xs-12 col-md-12">
            </div>
        </div>
 
        <!-- Lower middle column -->
        <div id="container_6" class="col-xs-12 col-md-4">
        </div>
 
        <!-- Lower right column -->
        <div class="col-xs-12 col-md-4">
            <div id="container_7" class="row_fluid">
                <div id="container_7_1" class="col-xs-12 col-md-6">
                </div>
                <div id="container_7_2" class="col-xs-12 col-md-6">
                </div>
                <div id="container_7_3" class="col-xs-12 col-md-12">
                </div>
            </div>
        </div>
 
    </div>

Echarts graphic code

var idContainer_4_5 = "container_4_5";
function initEchart_4_5() {
  // Initialize the ecarts instance based on the prepared dom
  var myChart = echarts.init(document.getElementById(idContainer_4_5), gTheme);
  option = {
    title: {
      text: "age distribution",
      top: "10%",
      left: "center",
      textStyle: {
        color: "#17c0ff",
        fontSize: "12",
      },
    },
 
    tooltip: {
      trigger: "item",
      formatter: "{a} <br/>{b}: {c} ({d}%)",
      position: function (p) {
        //Where p is the current mouse position
        return [p[0] + 10, p[1] - 10];
      },
    },
 
    grid: {
      left: "0",
      right: "10",
      bottom: "25%",
      top: "20%",
      containLabel: true,
    },
 
    xAxis: {
      type: "category",
      data: [],
      axisLabel: {
        textStyle: {
          color: "rgba(255,255,255,.8)",
          fontSize: 10,
        },
      },
      axisLine: {
        lineStyle: {
          color: "rgba(255,255,255,.2)",
        },
      },
      splitLine: {
        lineStyle: {
          color: "rgba(255,255,255,.1)",
        },
      },
    },
    yAxis: {
      type: "value",
      data: [],
      axisLabel: {
        textStyle: {
          color: "rgba(255,255,255,.8)",
          fontSize: 10,
        },
      },
      axisLine: {
        lineStyle: {
          color: "rgba(255,255,255,.2)",
        },
      },
      splitLine: {
        lineStyle: {
          color: "rgba(255,255,255,.1)",
        },
      },
    },
    series: [
      {
        name: "age distribution",
        type: "bar",
        stack: "total",
        label: {
          show: true,
        },
      },
    ],
  };
 
  // Use the configuration item and data you just specified to display the chart.
  myChart.setOption(option);
  window.addEventListener("resize", function () {
    myChart.resize();
  });
}
 
function getKeys(dataList) {
  var keys = [];
  var len = dataList.length;
  for (var i = 0; i < len; i++) keys.push(dataList[i].name);
  return keys;
}
 
function asyncData_4_5() {
  $.getJSON("myjson/bar_age.json").done(function (data) {
    var myChart = echarts.init(document.getElementById(idContainer_4_5));
    myChart.setOption({
      xAxis: { data: getKeys(data) },
      series: [{ data: data }],
    });
  }); //end $.getJSON
}
 
initEchart_4_5();

4, More cases

YYDataV's data visualization "wonderful case summary"_ YYDataV blog - CSDN blog ❤️ Data visualization ❤️: Large screen example based on Echarts + Vue [14]_ Little monster blog - CSDN blog_ Ecarts data visualization ❤️ Data visualization ❤️: Large screen example based on Echarts + Python [13] National Day Golden Week tourism monitoring is coming~_ Little monster blog - CSDN blog ❤️ Data visualization ❤️: Large screen example based on Echarts + Python implementation [12] (here's the cool world map you want!)_ Little devil's blog - CSDN blog data visualization: an example of dynamic real-time large screen based on Echarts + Python [11]https://yydatav.blog.csdn.net/article/details/120705616 At the end of this sharing, we welcome more exchanges. Please talk about business cooperation in private.

Keywords: Python Data Analysis Data Mining Flask echarts

Added by miracle_potential on Sun, 20 Feb 2022 18:23:55 +0200

Programming VIP