Configuration file syntax and settings

The configuration files passed as inputs to the kafka-connect-watcher are going to be evaluated and checked with jsonschema which validates the structure and validity of the input.

Below is the JSON schema definition, found in kafka_connect_watcher/watcher-config.spec.json

Kafka Connect Watcher configuration input

watcher-config.spec.json
Schema of the configuration for Kafka Connect Watcher
type	object
properties
clusters	List of Connect clusters to monitor.
	type	array
	items	#/definitions/ConnectCluster
	uniqueItems	True
prometheus
notification_channels	#/definitions/NotificationChannels
aws_emf	type	object
	properties
	log_group_name	override log group name to publish metrics to. Importance: High
		type	string
		default	kafka/connect/watcher/metrics
	service_name	override value for EMF Service name. Importance: Low
		type	string
	watcher_config	#/definitions/aws_emf
watch_interval	intervals converted to seconds between scans of all clusters.
	type	string
patternProperties
^x-
additionalProperties	False
definitions
ConnectCluster	type	object
	properties
	hostname	Connect cluster endpoint/IP Address
		type	string
	port	Connect cluster API Port
		type	integer
		default	8083
	url	The URL to the connect cluster, instead of hostname/port combination.
		type	string
	authentication	Basic Authentication
		#/definitions/BasicAuth
	ignore_https_error	Ignore SSL related errors (CA, self-signed)
		type	boolean
	metrics	Configure metrics export for the connect cluster
		#/definitions/ClusterMetrics
	error_handling_rules	List of rules (in-order) to execute if matching connectors state is not RUNNING
		type	array
		items	#/definitions/ErrorHandlingRule
		uniqueItems	True
ErrorHandlingRule	type	object
	properties
	include_regex	List of regular expressions or single regular expression to match and include for the error rule handling
		oneOf	type	string
			type	array
			items	type	string
	exclude_regex	List of regular expressions or single regular expression to exclude connectors for the error rule handling
		oneOf	type	string
			type	array
			items	type	string
	ignore_paused	If the connector is in PAUSED state, ignores it.
		type	boolean
		default	False
	auto_correct_actions	List (in-order) of actions to take in attempt to restore the connector.
		type	array
		items	#/definitions/AutoCorrectAction
	additionalProperties	False
AutoCorrectAction	type	object
	properties
	action	type	string
		enum	restart, pause, cycle, notify_only
	wait_for_status	duration to wait before checking on the connector status post action
		type	string
	notify	type	array
		items	#/definitions/AutoCorrectActionNotify
	on_action_failure	#/definitions/OnActionFailure
AutoCorrectActionNotify	type	object
	properties
	target	type	string
OnActionFailure	type	object
	properties
	log_level	type	string
		enum	INFO, ERROR, DEBUG, TRACE, WARN
ClusterMetrics	type	object
	properties
	prometheus	type	object
		properties
		enabled	type	boolean
	aws_emf	#/definitions/aws_emf
aws_emf	type	object
	properties
	namespace	type	string
	high_resolution_metrics	https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics
		type	boolean
		default	False
	dimensions	Dimension name and value to set in a key/value format.
		type	object
		patternProperties
		^\w+$	Value for the given dimension
			type	string
		maxProperties	25
		minProperties	1
	enabled	type	boolean
BasicAuth	type	object
	properties
	username	type	string
	password	type	string
	additionalProperties	False
NotificationChannels	type	object
	properties
	sns	type	object
		patternProperties
		^[a-zA-Z0-9-_]+$	#/definitions/SnsTopicChannel
	additionalProperties	False
SnsTopicChannel	type	object
	properties
	topic_arn	ARN of the SNS topic.
		type	string
	role_arn	Optional - Use IAM role to publish messages using another IAM role
		type	string
	ignore_errors	Prevents exception if true when an exception occurs.
		type	boolean
	template	Allows to set specific templates for email and sms
		type	object
		properties
		email	Optional - Path to a template for SNS Email messages
			type	string
		sms	Optional - Path to a template for SNS SMS messages
			type	string
		additionalProperties	False
	additionalProperties	False

Definition

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "id": "watcher-config.spec.json",
  "$id": "watcher-config.spec.json",
  "title": "Kafka Connect Watcher configuration input",
  "description": "Schema of the configuration for Kafka Connect Watcher",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "clusters": {
      "description": "List of Connect clusters to monitor.",
      "type": "array",
      "uniqueItems": true,
      "items": {
        "$ref": "#/definitions/ConnectCluster"
      }
    },
    "prometheus": {},
    "notification_channels": {
      "$ref": "#/definitions/NotificationChannels"
    },
    "aws_emf": {
      "type": "object",
      "properties": {
        "log_group_name": {
          "type": "string",
          "description": "override log group name to publish metrics to. Importance: High",
          "default": "kafka/connect/watcher/metrics"
        },
        "service_name": {
          "type": "string",
          "description": "override value for EMF Service name. Importance: Low"
        },
        "watcher_config": {
          "$ref": "#/definitions/aws_emf"
        }
      }
    },
    "watch_interval": {
      "type": "string",
      "description": "intervals converted to seconds between scans of all clusters."
    }
  },
  "definitions": {
    "ConnectCluster": {
      "type": "object",
      "required": [
        "hostname"
      ],
      "properties": {
        "hostname": {
          "description": "Connect cluster endpoint/IP Address",
          "type": "string"
        },
        "port": {
          "description": "Connect cluster API Port",
          "type": "integer",
          "default": 8083
        },
        "url": {
          "type": "string",
          "description": "The URL to the connect cluster, instead of hostname/port combination."
        },
        "authentication": {
          "description": "Basic Authentication",
          "$ref": "#/definitions/BasicAuth"
        },
        "ignore_https_error": {
          "type": "boolean",
          "description": "Ignore SSL related errors (CA, self-signed)"
        },
        "metrics": {
          "description": "Configure metrics export for the connect cluster",
          "$ref": "#/definitions/ClusterMetrics"
        },
        "error_handling_rules": {
          "type": "array",
          "uniqueItems": true,
          "description": "List of rules (in-order) to execute if matching connectors state is not RUNNING",
          "items": {
            "$ref": "#/definitions/ErrorHandlingRule"
          }
        }
      }
    },
    "ErrorHandlingRule": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "include_regex": {
          "description": "List of regular expressions or single regular expression to match and include for the error rule handling",
          "oneOf": [
            {
              "type": "string"
            },
            {
              "type": "array",
              "items": {
                "type": "string"
              }
            }
          ]
        },
        "exclude_regex": {
          "description": "List of regular expressions or single regular expression to exclude connectors for the error rule handling",
          "oneOf": [
            {
              "type": "string"
            },
            {
              "type": "array",
              "items": {
                "type": "string"
              }
            }
          ]
        },
        "ignore_paused": {
          "type": "boolean",
          "description": "If the connector is in PAUSED state, ignores it.",
          "default": false
        },
        "auto_correct_actions": {
          "description": "List (in-order) of actions to take in attempt to restore the connector.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/AutoCorrectAction"
          }
        }
      }
    },
    "AutoCorrectAction": {
      "type": "object",
      "required": [
        "action"
      ],
      "properties": {
        "action": {
          "type": "string",
          "enum": [
            "restart",
            "pause",
            "cycle",
            "notify_only"
          ]
        },
        "wait_for_status": {
          "type": "string",
          "description": "duration to wait before checking on the connector status post action"
        },
        "notify": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/AutoCorrectActionNotify"
          }
        },
        "on_action_failure": {
          "$ref": "#/definitions/OnActionFailure"
        }
      }
    },
    "AutoCorrectActionNotify": {
      "type": "object",
      "properties": {
        "target": {
          "type": "string"
        }
      }
    },
    "OnActionFailure": {
      "type": "object",
      "properties": {
        "log_level": {
          "type": "string",
          "enum": [
            "INFO",
            "ERROR",
            "DEBUG",
            "TRACE",
            "WARN"
          ]
        }
      }
    },
    "ClusterMetrics": {
      "type": "object",
      "properties": {
        "prometheus": {
          "type": "object",
          "properties": {
            "enabled": {
              "type": "boolean"
            }
          }
        },
        "aws_emf": {
          "$ref": "#/definitions/aws_emf"
        }
      }
    },
    "aws_emf": {
      "type": "object",
      "required": [
        "namespace"
      ],
      "properties": {
        "namespace": {
          "type": "string"
        },
        "high_resolution_metrics": {
          "type": "boolean",
          "default": false,
          "description": "https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics"
        },
        "dimensions": {
          "type": "object",
          "uniqueItems": true,
          "minProperties": 1,
          "maxProperties": 25,
          "description": "Dimension name and value to set in a key/value format.",
          "patternProperties": {
            "^\\w+$": {
              "type": "string",
              "description": "Value for the given dimension"
            }
          }
        },
        "enabled": {
          "type": "boolean"
        }
      }
    },
    "BasicAuth": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "username",
        "password"
      ],
      "properties": {
        "username": {
          "type": "string"
        },
        "password": {
          "type": "string"
        }
      }
    },
    "NotificationChannels": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "sns": {
          "type": "object",
          "uniqueItems": true,
          "patternProperties": {
            "^[a-zA-Z0-9-_]+$": {
              "$ref": "#/definitions/SnsTopicChannel"
            }
          }
        }
      }
    },
    "SnsTopicChannel": {
      "type": "object",
      "additionalProperties": false,
      "required": ["topic_arn"],
      "properties": {
        "topic_arn": {
          "type": "string",
          "description": "ARN of the SNS topic."
        },
        "role_arn": {
          "type": "string",
          "description": "Optional - Use IAM role to publish messages using another IAM role"
        },
        "ignore_errors": {
          "type": "boolean",
          "description": "Prevents exception if true when an exception occurs."
        },
        "template": {
          "type": "object",
          "description": "Allows to set specific templates for email and sms ",
          "additionalProperties": false,
          "properties": {
            "email": {
              "type": "string",
              "description": "Optional - Path to a template for SNS Email messages"
            },
            "sms": {
              "type": "string",
              "description": "Optional - Path to a template for SNS SMS messages"
            }
          }
        }
      }
    }
  },
  "patternProperties": {
    "^x-": {}
  }
}