Merge pull request #157 from lomik/rework-config

Rework configuration package

Author: Felixoid

.github/workflows/tests.yml

@@ -38,10 +38,16 @@ jobs:
         git fetch --shallow-since=$(git log $VERS -1 --format=%at)
       if: ${{ github.event_name == 'push' }}
 
-    - name: Test
+    - name: Build project
+      run: make
+
+    - name: Check documentation consistency
       run: |
-        make
-        make test
+        make config
+        git diff --exit-code
+
+    - name: Test
+      run: make test
 
     - name: Install packaging dependencies
       run: |

Makefile

@@ -19,14 +19,28 @@ all: $(NAME)
 
 .PHONY: clean
 clean:
+	rm $(NAME)
 	rm -rf out
 	rm -f *deb *rpm
 	rm -f sha256sum md5sum
 
-.PHONY: $(NAME)
-$(NAME):
+$(NAME): $(wildcard **/*.go)
 	$(GO) build $(MODULE)
 
+deploy/doc/graphite-clickhouse.conf: $(NAME)
+	./$(NAME) -config-print-default > $@
+
+doc/config.md: deploy/doc/graphite-clickhouse.conf deploy/doc/config.md
+	@echo 'Generating $@...'
+	@printf '[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually)  \n' > $@
+	@printf '[//]: # (To rebuild it run `make config`)\n\n' >> $@
+	@cat deploy/doc/config.md >> $@
+	@printf '\n```toml\n' >> $@
+	@cat deploy/doc/graphite-clickhouse.conf >> $@
+	@printf '```\n' >> $@
+
+config: doc/config.md
+
 test:
 	$(GO) test -race ./...
 

README.md

@@ -34,185 +34,7 @@ make
 3. Add graphite-clickhouse `host:port` to graphite-web [CLUSTER_SERVERS](http://graphite.readthedocs.io/en/latest/config-local-settings.html#cluster-configuration)
 
 ## Configuration
-Create `/etc/graphite-clickhouse/rollup.xml` with same content as for ClickHouse. Short sample:
-```xml
-<graphite_rollup>
-        <default>
-                <function>avg</function>
-                <retention>
-                        <age>0</age>
-                        <precision>60</precision>
-                </retention>
-                <retention>
-                        <age>2592000</age>
-                        <precision>3600</precision>
-                </retention>
-        </default>
-</graphite_rollup>
-```
-
-For complex ClickHouse queries you might need to increase default query_max_size. To do that add following line to `/etc/clickhouse-server/users.xml` for the user you are using:
-```xml
-<!-- Default is 262144 -->
-<max_query_size>10485760</max_query_size>
-```
-
-Create `/etc/graphite-clickhouse/graphite-clickhouse.conf`
-```toml
-[common]
-listen = ":9090"
-# Listener to serve /debug/pprof requests. `-pprof` argument would override it
-pprof-listen = ""
-max-cpu = 1
-# How frequently to call debug.FreeOSMemory() to return memory back to OS
-# Setting it to zero disables this functionality
-memory-return-interval = "0s"
-# Limit number of results from find query. Zero = unlimited
-max-metrics-in-find-answer = 0
-# Limit numbers of queried metrics per target in /render requests. Zero or negative are unlimited
-max-metrics-per-target = 15000
-# Daemon returns empty response if query matches any of regular expressions
-# target-blacklist = ["^not_found.*"]
-# If this > 0, then once an interval daemon will return the freed memory to the OS
-memory-return-interval = "0s"
-
-[clickhouse]
-# You can add user/password (http://user:password@localhost:8123) and any clickhouse options (GET-parameters) to url
-# It is recommended to create read-only user 
-url = "http://localhost:8123"
-# Add extra prefix (directory in graphite) for all metrics
-extra-prefix = ""
-
-# Default table with points
-data-table = "graphite"
-data-timeout = "1m0s"
-# Rollup rules xml filename. Use `auto` magic word for select rollup rules from ClickHouse
-rollup-conf = "/etc/graphite-clickhouse/rollup.xml"
-
-# Table with series list (daily and full)
-# https://github.com/lomik/graphite-clickhouse/wiki/IndexTable
-index-table = "graphite_index"
-# Use daily data from index table. This is useful for installations with big count of short-lived series but can be slower in other cases
-index-use-daily = true
-# Allow use reverse queries with minimal depth
-# -1 - disable
-# 0 - disable if no wildcard at first level
-# 1 - allow for example a.b.c*.d and a.b*.c.d queries (default)
-# 2 - allow a.b*.c.d) to index table.
-# This is useful when reverse queries has bad perfomance
-index-reverse-depth = 1
-# overwrite default index-use-reverse for metrics with prefix/suffix/regular_expression checks against target (not used when index-use-reverse = -1)
-#index-reverses = [
-#    { suffix = ".p99", reverse = 2 },
-#    { suffix = ".avg", reverse = 2 },
-#    { suffix = ".gc.gen1", reverse = 0 },
-#    { prefix = "Test.", suffix = ".cpu", reverse = 2 },
-#    { regex = "\\.gc\\.(heap|gen)[0-9]+$", reverse = 0 }
-#]
-index-reverses = []
-index-timeout = "1m0s"
-
-# `tagged` table from carbon-clickhouse. Required for seriesByTag
-tagged-table = ""
-# For how long the daemon will query tags during autocomplete
-tagged-autocomplete-days = 7
-
-# Old index tables. DEPRECATED
-tree-table = "graphite_tree"
-# Optional table with daily series list.
-# Useful for installations with big count of short-lived series
-date-tree-table = ""
-# Supported several schemas of date-tree-table:
-# 1 (default): table only with Path, Date, Level fields. Described here: https://habrahabr.ru/company/avito/blog/343928/
-# 2: table with Path, Date, Level, Deleted, Version fields. Table type "series" in the carbon-clickhouse
-# 3: same as #2 but with reversed Path. Table type "series-reverse" in the carbon-clickhouse
-date-tree-table-version = 0
-tree-timeout = "1m0s"
-
-connect-timeout = "1s"
-
-# Sets the maximum for maxDataPoints parameter.
-# If you use CH w/o https://github.com/ClickHouse/ClickHouse/pull/13947, you have to set it to 4096
-max-data-points = 4096
-# Use metrics aggregation on ClickHouse site.
-# This feature is very useful, read https://github.com/lomik/graphite-clickhouse/wiki/ClickHouse-aggregation-VS-graphite%E2%80%94clickhouse-aggregation
-internal-aggregation = false
-
-[prometheus]
-# The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for
-# generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all
-# HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically.
-external-url = ""
-page-title = "Prometheus Time Series Collection and Processing Server"
-
-[carbonlink]
-server = ""
-threads-per-request = 10
-connect-timeout = "50ms"
-query-timeout = "50ms"
-total-timeout = "500ms"
-
-# You can define multiple data tables (with points).
-# The first table that matches is used.
-#
-# # Sample, archived table with points older 30d
-# [[data-table]]
-# table = "graphite_archive"
-# min-age = "720h"
-# 
-# # All available options
-# [[data-table]]
-# # clickhouse table name
-# table = "table_name"
-# # points in table are stored with reverse path
-# reverse = false
-# # Custom rollup.xml for table. 
-# # Magic word `auto` can be used for load rules from ClickHouse
-# # With value `none` only rollup-default-precision and rollup-default-function will be used for rollup
-# rollup-conf = ""
-# # Which table to discover rollup-rules from. If not specified - will use what specified in "table" parameter.
-# # Useful when reading from distributed table, but the rollup parameters are on the shard tables.
-# # Can be in "database.table" form.
-# rollup-auto-table = ""
-# # Use unreversed metric name to discover rollup-rules. If not specified - use as is (reversed metric name).
-# # Use if unreversed rules is used.
-# # Ignored for non-reverse tables (unreversed metric names used always)
-# rollup-use-reverted = false
-# # Sets the default precision and function for rollup patterns which don't have age=0 retention defined.
-# # If age=0 retention is defined in the rollup config then it takes precedence.
-# # If left at the default value of 0 then no rollup is performed when the requested interval 
-# # is not covered by any rollup rule. In this case the points will be served with 60 second precision.
-# rollup-default-precision = 60
-# rollup-default-function = "avg"
-# # from >= now - {max-age}
-# max-age = "240h"
-# # until <= now - {min-age}
-# min-age = "240h"
-# # until - from <= {max-interval}
-# max-interval = "24h"
-# # until - from >= {min-interval}
-# min-interval = "24h"
-# # regexp.Match({target-match-any}, target[0]) || regexp.Match({target-match-any}, target[1]) || ...
-# target-match-any = "regexp"
-# # regexp.Match({target-match-all}, target[0]) && regexp.Match({target-match-all}, target[1]) && ...
-# target-match-all = "regexp"
-
-[debug]
-# The directory for debug info. If set, additional info may be saved there
-directory = "/var/log/graphite-clickhouse/debug"
-directory-perm = "0755"
-# File permissions for external data dumps. Enabled only if !=0, see X-Gch-Debug-External-Data header
-# Format is octal, e.g. 0640
-external-data-perm = "0644"
-
-[[logging]]
-logger = ""
-file = "/var/log/graphite-clickhouse/graphite-clickhouse.log"
-level = "info"
-encoding = "mixed"
-encoding-time = "iso8601"
-encoding-duration = "seconds"
-```
+See [configuration documentation](./doc/config.md).
 
 ### Special headers processing
 

autocomplete/autocomplete.go

@@ -142,11 +142,11 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 
 	body, err := clickhouse.Query(
 		scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
-		h.config.ClickHouse.Url,
+		h.config.ClickHouse.URL,
 		sql,
 		clickhouse.Options{
-			Timeout:        h.config.ClickHouse.TreeTimeout.Value(),
-			ConnectTimeout: h.config.ClickHouse.ConnectTimeout.Value(),
+			Timeout:        h.config.ClickHouse.IndexTimeout,
+			ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
 		},
 		nil,
 	)
@@ -248,11 +248,11 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 
 	body, err := clickhouse.Query(
 		scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
-		h.config.ClickHouse.Url,
+		h.config.ClickHouse.URL,
 		sql,
 		clickhouse.Options{
-			Timeout:        h.config.ClickHouse.IndexTimeout.Value(),
-			ConnectTimeout: h.config.ClickHouse.ConnectTimeout.Value(),
+			Timeout:        h.config.ClickHouse.IndexTimeout,
+			ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
 		},
 		nil,
 	)

config/.gitignore

@@ -0,0 +1 @@
+tests_tmp/