Merge branch 'main' into stderr-utf8

* main:
  Command fix. Must be list append
  Refactorings
  Moved tinyproxy out of if clause
  Refactoring for error messages and security fix for path echoing (#636)
  GMT color via own commit hash (#634)
  Hotfix for branch not main
  Non-Blocking starlette body read (#633)
  Bump fastapi from 0.105.0 to 0.108.0 (#632)
  Updated XGBoost submodule
  Bump pydantic from 2.5.2 to 2.5.3 (#628)
  Added stddev to timeline (#627)

Author: ArneTR

api/api_helpers.py

@@ -125,7 +125,7 @@ def get_timeline_query(uri, filename, machine_id, branch, metrics, phase, start_
     if filename is None or filename.strip() == '':
         filename =  'usage_scenario.yml'
 
-    if branch is None or branch.strip() != '':
+    if branch is None or branch.strip() == '':
         branch = 'main'
 
     params = [uri, filename, branch, machine_id, f"%{phase}"]
@@ -163,7 +163,7 @@ def get_timeline_query(uri, filename, machine_id, branch, metrics, phase, start_
     query = f"""
             SELECT
                 r.id, r.name, r.created_at, p.metric, p.detail_name, p.phase,
-                p.value, p.unit, r.commit_hash, r.commit_timestamp,
+                p.value, p.unit, r.commit_hash, r.commit_timestamp, r.gmt_hash,
                 row_number() OVER () AS row_num
             FROM runs as r
             LEFT JOIN phase_stats as p ON

api/main.py

@@ -84,25 +84,21 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
 
 @app.exception_handler(StarletteHTTPException)
 async def http_exception_handler(request, exc):
-    await log_exception(request, exc, body='StarletteHTTPException handler cannot read body atm. Waiting for FastAPI upgrade.', details=exc.detail)
+    body = await request.body()
+    await log_exception(request, exc, body=body, details=exc.detail)
     return ORJSONResponse(
         status_code=exc.status_code,
         content=jsonable_encoder({'success': False, 'err': exc.detail}),
     )
 
 async def catch_exceptions_middleware(request: Request, call_next):
     #pylint: disable=broad-except
+    body = None
     try:
+        body = await request.body()
         return await call_next(request)
     except Exception as exc:
-        # body = await request.body()  # This blocks the application. Unclear atm how to handle it properly
-        # seems like a bug: https://github.com/tiangolo/fastapi/issues/394
-        # Although the issue is closed the "solution" still behaves with same failure
-        # Actually Starlette, the underlying library to FastAPI has already introduced this functionality:
-        # https://github.com/encode/starlette/pull/1692
-        # However FastAPI does not support the new Starlette 0.31.1
-        # The PR relevant here is: https://github.com/tiangolo/fastapi/pull/9939
-        await log_exception(request, exc, body='Middleware cannot read body atm. Waiting for FastAPI upgrade')
+        await log_exception(request, exc, body=body)
         return ORJSONResponse(
             content={
                 'success': False,

docker/requirements.txt

@@ -1,6 +1,7 @@
 gunicorn==21.2.0
 psycopg[binary]==3.1.16
-fastapi==0.105.0
+fastapi==0.108.0
+starlette>=0.32
 uvicorn[standard]==0.25.0
 pandas==2.1.4
 PyYAML==6.0.1

frontend/js/helpers/charts.js

@@ -1,3 +1,9 @@
+const calculateStatistics = (data) => {
+    const mean = data.reduce((sum, value) => sum + value.value, 0) / data.length;
+    const stddev = Math.sqrt(data.reduce((sum, value) => sum + Math.pow(value.value - mean, 2), 0) / data.length);
+    return { mean, stddev };
+}
+
 const getCompareChartOptions = (legend, series, chart_type='line', x_axis='time', y_axis_name, mark_area=null,  graphic=null) => {
     let tooltip_trigger = (chart_type=='line') ? 'axis' : 'item';
 
@@ -106,7 +112,7 @@ const calculateMA = (series, factor) => {
   return result;
 }
 
-const getLineBarChartOptions = (legend, labels, series, x_axis_name=null, y_axis_name='', x_axis='time', mark_area=null, no_toolbox=false, graphic=null, moving_average=false, show_x_axis_label=true) => {
+const getLineBarChartOptions = (legend, labels, series, x_axis_name=null, y_axis_name='', x_axis='time', mark_area=null, no_toolbox=false, graphic=null, moving_average=false, show_x_axis_label=true, stddev=false) => {
 
     if(Object.keys(series).length == 0) {
         return {graphic: getChartGraphic("No energy reporter active")};
@@ -128,6 +134,31 @@ const getLineBarChartOptions = (legend, labels, series, x_axis_name=null, y_axis
        })
    }
 
+   if(stddev) {
+       const { mean, stddev } = calculateStatistics(series[0].data);
+
+       legend.push('Stddev')
+       series.push({
+            name: 'Stddev',
+            type: 'line',
+            markArea: {
+                label: {
+                    show: true,
+                    name: "MarkArea",
+                    position: 'top'
+                },
+                data: [
+                    [
+                        { yAxis: mean + stddev, name: `StdDev: ${(stddev/mean * 100).toFixed(2)} %`},
+                        { yAxis: mean - stddev},
+                    ]
+
+                ],
+            }
+        });
+   }
+
+
    let options =  {
         tooltip: { trigger: tooltip_trigger },
         grid: {

frontend/js/timeline.js

@@ -39,13 +39,13 @@ function* colorIterator() {
     }
 }
 
-const generateColoredValues = (values) => {
+const generateColoredValues = (values, key) => {
     const color_iterator = colorIterator()
-    let last_commit_hash = null
+    let last_hash = null
     let color = null;
     return values.map((value) => {
-        if(last_commit_hash != value.commit_hash) {
-            last_commit_hash = value.commit_hash
+        if(last_hash != value[key]) {
+            last_hash = value[key]
             color = color_iterator.next().value
         }
         return {value: value.value, itemStyle: {color: color}}
@@ -159,15 +159,15 @@ const loadCharts = async () => {
     let prun_id = null
 
     phase_stats_data.forEach( (data) => {
-        let [run_id, run_name, created_at, metric_name, detail_name, phase, value, unit, commit_hash, commit_timestamp] = data
+        let [run_id, run_name, created_at, metric_name, detail_name, phase, value, unit, commit_hash, commit_timestamp, gmt_hash] = data
 
 
         if (series[`${metric_name} - ${detail_name}`] == undefined) {
             series[`${metric_name} - ${detail_name}`] = {labels: [], values: [], notes: [], unit: unit, metric_name: metric_name, detail_name: detail_name}
         }
 
         series[`${metric_name} - ${detail_name}`].labels.push(commit_timestamp)
-        series[`${metric_name} - ${detail_name}`].values.push({value: value, commit_hash: commit_hash})
+        series[`${metric_name} - ${detail_name}`].values.push({value: value, commit_hash: commit_hash, gmt_hash: gmt_hash})
         series[`${metric_name} - ${detail_name}`].notes.push({
             run_name: run_name,
             created_at: created_at,
@@ -176,6 +176,7 @@ const loadCharts = async () => {
             phase: phase,
             run_id: run_id,
             prun_id: prun_id,
+            gmt_hash: gmt_hash,
         })
 
         prun_id = run_id
@@ -203,7 +204,7 @@ const loadCharts = async () => {
 
         const chart_instance = echarts.init(element);
 
-        const my_values = generateColoredValues(series[my_series].values);
+        const my_values = generateColoredValues(series[my_series].values, $('.radio-coloring:checked').val());
 
         let data_series = [{
             name: my_series,
@@ -218,34 +219,53 @@ const loadCharts = async () => {
             }
         }]
 
-        let options = getLineBarChartOptions([], series[my_series].labels, data_series, 'Time', series[my_series].unit,  'category', null, false, null, true, false);
+        let options = getLineBarChartOptions([], series[my_series].labels, data_series, 'Time', series[my_series].unit,  'category', null, false, null, true, false, true);
 
         options.tooltip = {
-            trigger: 'item',
+            triggerOn: 'click',
             formatter: function (params, ticket, callback) {
-                if(params.componentType != 'series') return; // no notes for the MovingAverage
+                if(series[params.seriesName]?.notes == null) return; // no notes for the MovingAverage
                 return `<strong>${series[params.seriesName].notes[params.dataIndex].run_name}</strong><br>
+                        run_id: <a href="/stats.html?id=${series[params.seriesName].notes[params.dataIndex].run_id}"  target="_blank">${series[params.seriesName].notes[params.dataIndex].run_id}</a><br>
                         date: ${series[params.seriesName].notes[params.dataIndex].created_at}<br>
                         metric_name: ${params.seriesName}<br>
                         phase: ${series[params.seriesName].notes[params.dataIndex].phase}<br>
                         value: ${numberFormatter.format(series[params.seriesName].values[params.dataIndex].value)}<br>
                         commit_timestamp: ${series[params.seriesName].notes[params.dataIndex].commit_timestamp}<br>
-                        commit_hash: ${series[params.seriesName].notes[params.dataIndex].commit_hash}<br>
+                        commit_hash: <a href="${$("#uri").text()}/commit/${series[params.seriesName].notes[params.dataIndex].commit_hash}" target="_blank">${series[params.seriesName].notes[params.dataIndex].commit_hash}</a><br>
+                        gmt_hash: <a href="https://github.com/green-coding-berlin/green-metrics-tool/commit/${series[params.seriesName].notes[params.dataIndex].gmt_hash}" target="_blank">${series[params.seriesName].notes[params.dataIndex].gmt_hash}</a><br>
+
                         <br>
-                        <i>Click to diff measurement with previous</i>
+                        👉 <a href="/compare.html?ids=${series[params.seriesName].notes[params.dataIndex].run_id},${series[params.seriesName].notes[params.dataIndex].prun_id}" target="_blank">Diff with previous run</a>
                         `;
             }
         };
 
-        chart_instance.on('click', function (params) {
-            if(params.componentType != 'series') return; // no notes for the MovingAverage
-            window.open(`/compare.html?ids=${series[params.seriesName].notes[params.dataIndex].run_id},${series[params.seriesName].notes[params.dataIndex].prun_id}`, '_blank');
-
-        });
+        options.dataZoom = {
+            show: false,
+            start: 0,
+            end: 100,
+        };
 
 
         chart_instance.setOption(options);
         chart_instances.push(chart_instance);
+        chart_instance.on('datazoom', function(e, f) {
+            const data = chart_instance.getOption().series[0].data
+            const dataZoomOption = chart_instance.getOption().dataZoom[0];
+            const startPercent = dataZoomOption.start;
+            const endPercent = dataZoomOption.end;
+            const totalDataPoints = data.length;
+            const startIndex = Math.floor(startPercent / 100 * totalDataPoints);
+            const endIndex = Math.ceil(endPercent / 100 * totalDataPoints) - 1;
+            const { mean, stddev } = calculateStatistics(data.slice(startIndex, endIndex+1));
+
+            let options = chart_instance.getOption()
+            options.series[2].markArea.data[0][0].name = `StdDev: ${(stddev/mean * 100).toFixed(2)} %`
+            options.series[2].markArea.data[0][0].yAxis = mean + stddev
+            options.series[2].markArea.data[0][1].yAxis = mean - stddev;
+            chart_instance.setOption(options)
+        });
 
     }
 

frontend/timeline.html

@@ -37,6 +37,7 @@
     <link rel="stylesheet" type="text/css" href="/css/green-coding.css">
     <style type="text/css">
         .hide-for-single-stats { display: none !important; }
+        .statistics-chart div { pointer-events: all !important }
     </style>
 </head>
 <body class="preload">
@@ -195,6 +196,21 @@ <h4>What is Timeline View?</h4>
                                     <i class="info circle icon"></i> Will not affect badge
                                 </div>
                             </div>
+                            <div class="inline fields">
+                                <label>Coloring:</label>
+                                <div class="field">
+                                    <div class="ui radio checkbox">
+                                        <input class="radio-coloring" id="coloring-commit_hash" type="radio" name="coloring" value="commit_hash" checked>
+                                        <label for="coloring-commit_hash">Software commit hash</label>
+                                    </div>
+                                </div>
+                                <div class="field">
+                                    <div class="ui radio checkbox">
+                                        <input class="radio-coloring" id="coloring-gmt_hash" type="radio" name="coloring" value="gmt_hash">
+                                        <label for="coloring-gmt_hash">GMT commit hash</label>
+                                    </div>
+                                </div>
+                            </div>
                             <div class="three fields">
                                 <div class="field">
                                     <label>Start date</label>
@@ -233,7 +249,8 @@ <h4>What is Timeline View?</h4>
                 <div class="header">
                     Graph Info
                 </div>
-                <p>Graphs show every measurement as a bar. We color the bars according to the commit timestamp and change the color for every new commit. After some time the color repeats ...</p>
+                <p>Graphs show every measurement as a bar. We color the bars according to the commit timestamp and change the color for every new commit (You can also change to color by the GMT tool hash). After some time the color repeats ...</p>
+                <p>To show details click on the bars! A new menu will appear.</p>
             </div>
         </div>
         <div class="ui two cards" id="api-loader">

install_linux.sh

@@ -117,14 +117,12 @@ git submodule update --init
 print_message "Installing needed binaries for building ..."
 if lsb_release -is | grep -q "Fedora"; then
     sudo dnf -y install lm_sensors lm_sensors-devel glib2 glib2-devel tinyproxy lshw
-    sudo systemctl stop tinyproxy
-    sudo systemctl disable tinyproxy
 else
     sudo apt-get update
     sudo apt-get install -y lm-sensors libsensors-dev libglib2.0-0 libglib2.0-dev tinyproxy lshw
-    sudo systemctl stop tinyproxy
-    sudo systemctl disable tinyproxy
 fi
+sudo systemctl stop tinyproxy
+sudo systemctl disable tinyproxy
 
 print_message "Building binaries ..."
 metrics_subdir="metric_providers"

lib/error_helpers.py

@@ -11,14 +11,14 @@ def end_error(*errors):
 
 
 def format_error(*errors):
-    err = 'Error: '
+    err = ''
 
     for error in errors:
-        err += str(error)
+        err += str(error) + "\n"
 
     error_string = f"""
         \n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n
-        {err}
+         Error: {err}
         \n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n
         {traceback.format_exc()}
         \n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n
@@ -30,11 +30,15 @@ def format_error(*errors):
 def log_error(*errors):
     error_log_file = GlobalConfig().config['machine']['error_log_file']
 
+    err = ''
+    for error in errors:
+        err += str(error) + "\n"
+
     if error_log_file:
         try:
             with open(error_log_file, 'a', encoding='utf-8') as file:
                 print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=file)
-                print('Error: ', *errors, file=file)
+                print('Error: ', err, file=file)
                 print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=file)
                 print(traceback.format_exc(), file=file)
                 print('\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=file)
@@ -46,6 +50,6 @@ def log_error(*errors):
           '\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=sys.stderr)
     print(traceback.format_exc(), file=sys.stderr)
     print('\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=sys.stderr)
-    print('Error: ', *errors, file=sys.stderr)
+    print('Error: ', err, file=sys.stderr)
     print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n',
           TerminalColors.ENDC, file=sys.stderr)

metric_providers/psu/energy/ac/xgboost/machine/model

@@ -1,9 +1,10 @@
 -r requirements.txt
-pydantic==2.5.2
+pydantic==2.5.3
 pytest==7.4.3
 requests==2.31.0
 pylint==3.0.3
-fastapi==0.105.0
+fastapi==0.108.0
+starlette>=0.32
 anybadge==1.14.0
 
 # just to clear the pylint errors for the files in /api