Limit some high cardinality metrics by default

Author: smarterclayton

examples/prometheus/prometheus.yaml

@@ -255,6 +255,7 @@ objects:
             miqTarget: "ContainerNode"
             severity: "HIGH"
             message: "{{$labels.instance}} is down"
+    
     recording.rules: |
       groups:
       - name: aggregate_container_resources
@@ -267,6 +268,7 @@ objects:
           expr: sum by (hostname,type)(rate(container_cpu_usage_seconds_total{id="/"}[3m])) / on (hostname,type) machine_cpu_cores
         - record: apiserver_request_count_by_resources
           expr: sum without (client,instance,contentType) (rate(apiserver_request_count[5m]))
+
     prometheus.yml: |
       rule_files:
         - '*.rules'
@@ -368,6 +370,14 @@ objects:
         kubernetes_sd_configs:
         - role: node
 
+        # Exclude a set of high cardinality metrics that can contribute to significant
+        # memory use in large clusters. These can be selectively enabled as necessary
+        # for medium or small clusters.
+        metric_relabel_configs:
+        - source_labels: [__name__]
+          action: drop
+          regex: 'container_(cpu_user_seconds_total|cpu_cfs_periods_total|memory_usage_bytes|memory_swap|memory_working_set_bytes|memory_cache|last_seen|fs_(read_seconds_total|write_seconds_total|sector_(.*)|io_(.*)|reads_merged_total|writes_merged_total)|tasks_state|memory_failcnt|memory_failures_total|spec_memory_swap_limit_bytes|fs_(.*)_bytes_total|spec_(.*))|openshift_sdn_pod_(setup|teardown)_(.*)'
+
         relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)