chore(outputs.sql): Make default create table template suitable for ClickHouse

AndreKR · AndreKR · commit 80069ac8cce5 · 2025-01-26T22:13:27.000+01:00
This changes the default TableTemplate of the SQL output plugin so that
it can be used with ClickHouse. Previously it was invalid for ClickHouse
because it didn't include a PRIMARY KEY or ORDER BY clause, which is
required for ClickHouse.

This then also allows to remove the custom TableTemplate from the
ClickHouse unit test.

This also adds another placeholder {TAG_COLUMNS} that allows to get a
list of all tag columns in a TableTemplate. This can then be used to use
the tag columns as a primary or sort key, especially in ClickHouse.
diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md
@@ -99,6 +99,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf
@@ -18,6 +18,7 @@
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go
@@ -146,13 +146,15 @@ func (p *SQL) deriveDatatype(value interface{}) string {
 
 func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	columns := make([]string, 0, len(metric.TagList())+len(metric.FieldList())+1)
+	tagColumns := make([]string, 0, len(metric.TagList()))
 
 	if p.TimestampColumn != "" {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(p.TimestampColumn), p.Convert.Timestamp))
 	}
 
 	for _, tag := range metric.TagList() {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(tag.Key), p.Convert.Text))
+		tagColumns = append(tagColumns, quoteIdent(tag.Key))
 	}
 
 	var datatype string
@@ -165,6 +167,13 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	query = strings.ReplaceAll(query, "{TABLE}", quoteIdent(metric.Name()))
 	query = strings.ReplaceAll(query, "{TABLELITERAL}", quoteStr(metric.Name()))
 	query = strings.ReplaceAll(query, "{COLUMNS}", strings.Join(columns, ","))
+	query = strings.ReplaceAll(query, "{TAG_COLUMNS}", strings.Join(tagColumns, ","))
+
+	if p.Driver == "clickhouse" {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", fmt.Sprintf("ORDER BY (%s, %s)", strings.Join(tagColumns, ","), p.TimestampColumn))
+	} else {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", "")
+	}
 
 	return query
 }
@@ -273,7 +282,7 @@ func init() {
 
 func newSQL() *SQL {
 	return &SQL{
-		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS})",
+		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS}) {SORT_KEY_CLAUSE}",
 		TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1",
 		TimestampColumn:     "timestamp",
 		Convert: ConvertStruct{
diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go
@@ -368,7 +368,6 @@ func TestClickHouseIntegration(t *testing.T) {
 	p.Log = testutil.Logger{}
 	p.Driver = "clickhouse"
 	p.DataSourceName = address
-	p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp"
 	p.Convert.Integer = "Int64"
 	p.Convert.Text = "String"
 	p.Convert.Timestamp = "DateTime"
