Improve Azure Managed Instance support + more in sqlserver input (#4642)

Gopkg.lock

@@ -1,6 +1,14 @@
 # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
 
 
+[[projects]]
+  digest = "1:972f38a9c879a4920d1e3a3d3438104b6c06163bfa3e6f4064adb00468d40587"
+  name = "cloud.google.com/go"
+  packages = ["civil"]
+  pruneopts = ""
+  revision = "c728a003b238b26cef9ab6753a5dc424b331c3ad"
+  version = "v0.27.0"
+
 [[projects]]
   branch = "master"
   digest = "1:fc0802104acded1f48e4860a9f2db85b82b4a754fca9eae750ff4e8b8cdf2116"
@@ -122,7 +130,7 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:0828d8c0f95689f832cf348fe23827feb7640cd698d612ef59e2f9d041f54c68"
+  digest = "1:5a5f28fcfe3a74247733a31ceaac0e53bfc2723e43c596b2e3f110eda9378575"
   name = "github.com/apache/thrift"
   packages = ["lib/go/thrift"]
   pruneopts = ""
@@ -230,6 +238,17 @@
   revision = "346938d642f2ec3594ed81d874461961cd0faa76"
   version = "v1.1.0"
 
+[[projects]]
+  branch = "master"
+  digest = "1:7fdc54859cd901c25b9d8db964410a4e0d98fa0dca267fe4cf49c0eede5e06c2"
+  name = "github.com/denisenkom/go-mssqldb"
+  packages = [
+    ".",
+    "internal/cp",
+  ]
+  pruneopts = ""
+  revision = "1eb28afdf9b6e56cf673badd47545f844fe81103"
+
 [[projects]]
   digest = "1:6098222470fe0172157ce9bbef5d2200df4edde17ee649c5d6e48330e4afa4c6"
   name = "github.com/dgrijalva/jwt-go"
@@ -257,7 +276,7 @@
   revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c"
 
 [[projects]]
-  digest = "1:d149605f1b00713fdc48150122892d77d49d30c825f690dd92f497aeb6cf18f5"
+  digest = "1:d2ca9295cce7d0e7b26b498c6b59ff903d8315e8ead97f0f6cadf9e7d613e1e8"
   name = "github.com/docker/docker"
   packages = [
     "api",
@@ -498,7 +517,7 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:ff65bf6fc4d1116f94ac305342725c21b55c16819c2606adc8f527755716937f"
+  digest = "1:e1c91a91cc738cebecbf12fc98f554f6f932c8b97e2052ad63ea43948df5bcb0"
   name = "github.com/hashicorp/go-rootcerts"
   packages = ["."]
   pruneopts = ""
@@ -1016,14 +1035,6 @@
   pruneopts = ""
   revision = "46796da1b0b4794e1e341883a399f12cc7574b55"
 
-[[projects]]
-  branch = "master"
-  digest = "1:2fcfc6c3fb8dfe0d80d7789272230d3ac7db15022b66817113f98d9fff880225"
-  name = "github.com/zensqlmonitor/go-mssqldb"
-  packages = ["."]
-  pruneopts = ""
-  revision = "e8fbf836e44e86764eba398361d1825651709547"
-
 [[projects]]
   branch = "master"
   digest = "1:0773b5c3be42874166670a20aa177872edb450cd9fc70b1df97303d977702a50"
@@ -1312,6 +1323,7 @@
     "github.com/aws/aws-sdk-go/service/sts",
     "github.com/bsm/sarama-cluster",
     "github.com/couchbase/go-couchbase",
+    "github.com/denisenkom/go-mssqldb",
     "github.com/dgrijalva/jwt-go",
     "github.com/docker/docker/api/types",
     "github.com/docker/docker/api/types/container",
@@ -1375,7 +1387,6 @@
     "github.com/vmware/govmomi/vim25/soap",
     "github.com/vmware/govmomi/vim25/types",
     "github.com/wvanbergen/kafka/consumergroup",
-    "github.com/zensqlmonitor/go-mssqldb",
     "golang.org/x/net/context",
     "golang.org/x/net/html/charset",
     "golang.org/x/oauth2",

Gopkg.toml

@@ -183,7 +183,7 @@
   branch = "master"
 
 [[constraint]]
-  name = "github.com/zensqlmonitor/go-mssqldb"
+  name = "github.com/denisenkom/go-mssqldb"
   branch = "master"
 
 [[constraint]]

plugins/inputs/sqlserver/README.md

@@ -90,7 +90,17 @@ The new (version 2) metrics provide:
   - *TempDB*: Free space, Version store usage, Active temp tables, temp table creation rate, + more
   - *Resource Governor*: CPU Usage, Requests/sec, Queued Requests, and Blocked tasks per workload group + more
 - *Server properties*: Number of databases in all possible states (online, offline, suspect, etc.), cpu count, physical memory, SQL Server service uptime, and SQL Server version
-- *Wait stats*: Wait time in ms, number of waiting tasks, resource wait time, signal wait time, max wait time in ms, wait type, and wait category. The waits are categorized using the sasme categories used in Query Store.
+- *Wait stats*: Wait time in ms, number of waiting tasks, resource wait time, signal wait time, max wait time in ms, wait type, and wait category. The waits are categorized using the same categories used in Query Store.
+- *Azure Managed Instances*
+  - Stats from `sys.server_resource_stats`:
+    - cpu_count
+    - server_memory
+    - sku
+    - engine_edition
+    - hardware_type
+    - total_storage_mb
+    - available_storage_mb
+    - uptime
 
 The following metrics can be used directly, with no delta calculations:
  - SQLServer:Buffer Manager\Buffer cache hit ratio
@@ -129,5 +139,4 @@ The following metrics can be used directly, with no delta calculations:
  - SQLServer:Workload Group Stats\Requests completed/sec
 
 Version 2 queries have the following tags:
-- `sql_instance`: Physical host and instance name (hostname:instance)
+- `sql_instance`: Physical host and instance name (hostname:instance)
-

plugins/inputs/sqlserver/sqlserver.go

@@ -9,7 +9,7 @@ import (
 	"github.com/influxdata/telegraf/plugins/inputs"
 
 	// go-mssqldb initialization
-	_ "github.com/zensqlmonitor/go-mssqldb"
+	_ "github.com/denisenkom/go-mssqldb"
 )
 
 // SQLServer struct
@@ -348,7 +348,9 @@ ELSE
 EXEC(@SQL)
 `
 
-const sqlDatabaseIOV2 = `SELECT
+const sqlDatabaseIOV2 = `IF SERVERPROPERTY('EngineEdition') = 5
+BEGIN
+SELECT
 'sqlserver_database_io' As [measurement],
 REPLACE(@@SERVERNAME,'\',':') AS [sql_instance],
 DB_NAME([vfs].[database_id]) [database_name],
@@ -358,43 +360,123 @@ vfs.num_of_bytes_read AS read_bytes,
 vfs.io_stall_write_ms AS write_latency_ms,
 vfs.num_of_writes AS writes,
 vfs.num_of_bytes_written AS write_bytes,
-CASE WHEN vfs.file_id = 2 THEN 'LOG' ELSE 'ROWS' END AS file_type
+b.name as logical_filename,
+b.physical_name as physical_filename,
+CASE WHEN vfs.file_id = 2 THEN 'LOG' ELSE 'DATA' END AS file_type
 FROM
 [sys].[dm_io_virtual_file_stats](NULL,NULL) AS vfs
-OPTION( RECOMPILE );
+inner join sys.database_files b on  b.file_id = vfs.file_id
+END
+ELSE
+BEGIN
+SELECT
+'sqlserver_database_io' As [measurement],
+REPLACE(@@SERVERNAME,'\',':') AS [sql_instance],
+DB_NAME([vfs].[database_id]) [database_name],
+vfs.io_stall_read_ms AS read_latency_ms,
+vfs.num_of_reads AS reads,
+vfs.num_of_bytes_read AS read_bytes,
+vfs.io_stall_write_ms AS write_latency_ms,
+vfs.num_of_writes AS writes,
+vfs.num_of_bytes_written AS write_bytes,
+b.name as logical_filename,
+b.physical_name as physical_filename,
+CASE WHEN vfs.file_id = 2 THEN 'LOG' ELSE 'DATA' END AS file_type
+FROM
+[sys].[dm_io_virtual_file_stats](NULL,NULL) AS vfs
+inner join sys.master_files b on b.database_id = vfs.database_id and b.file_id = vfs.file_id
+END
 `
 
 const sqlServerPropertiesV2 = `DECLARE @sys_info TABLE (
 	cpu_count INT,
-	server_memory INT,
+	server_memory BIGINT,
+	sku NVARCHAR(64),
+	engine_edition SMALLINT,
+	hardware_type VARCHAR(15),
+	total_storage_mb BIGINT,
+	available_storage_mb BIGINT,
 	uptime INT
 )
 
 IF OBJECT_ID('master.sys.dm_os_sys_info') IS NOT NULL
 BEGIN
-	INSERT INTO @sys_info ( cpu_count, server_memory, uptime )
+ 
-	EXEC('SELECT cpu_count, (select total_physical_memory_kb from sys.dm_os_sys_memory) AS physical_memory_kb, DATEDIFF(MINUTE,sqlserver_start_time,GETDATE()) FROM sys.dm_os_sys_info')
+	IF SERVERPROPERTY('EngineEdition') = 8  -- Managed Instance
-END
+		INSERT INTO @sys_info ( cpu_count, server_memory, sku, engine_edition, hardware_type, total_storage_mb, available_storage_mb, uptime )
+		SELECT 	TOP(1)
+				virtual_core_count AS cpu_count,
+				(SELECT process_memory_limit_mb FROM sys.dm_os_job_object) AS server_memory,
+				sku,
+				cast(SERVERPROPERTY('EngineEdition') as smallint) AS engine_edition,
+				hardware_generation AS hardware_type, 
+				reserved_storage_mb AS total_storage_mb, 
+				(reserved_storage_mb - storage_space_used_mb) AS available_storage_mb,
+				(select DATEDIFF(MINUTE,sqlserver_start_time,GETDATE()) from sys.dm_os_sys_info) as uptime
+		FROM	sys.server_resource_stats
+		ORDER BY start_time DESC
 
-SELECT
+	ELSE
-'sqlserver_server_properties' As [measurement],
+	BEGIN
-REPLACE(@@SERVERNAME,'\',':') AS [sql_instance],
+		DECLARE @total_disk_size_mb BIGINT,
-SUM( CASE WHEN state = 0 THEN 1 ELSE 0 END ) AS db_online,
+				@available_space_mb BIGINT
-SUM( CASE WHEN state = 1 THEN 1 ELSE 0 END ) AS db_restoring,
+
-SUM( CASE WHEN state = 2 THEN 1 ELSE 0 END ) AS db_recovering,
+		SELECT	@total_disk_size_mb = sum(total_disk_size_mb),
-SUM( CASE WHEN state = 3 THEN 1 ELSE 0 END ) AS db_recoveryPending,
+				@available_space_mb = sum(free_disk_space_mb) 
-SUM( CASE WHEN state = 4 THEN 1 ELSE 0 END ) AS db_suspect,
+		FROM	(
-SUM( CASE WHEN state = 10 THEN 1 ELSE 0 END ) AS db_offline,
+					SELECT	distinct logical_volume_name AS LogicalName,
-MAX( sinfo.cpu_count ) AS cpu_count,
+							total_bytes/(1024*1024)as total_disk_size_mb, 
-MAX( sinfo.server_memory ) AS server_memory,
+							available_bytes /(1024*1024) free_disk_space_mb
-MAX( sinfo.uptime ) AS uptime,
+					FROM	sys.master_files AS f  
-SERVERPROPERTY('ProductVersion') AS sql_version
+							CROSS APPLY sys.dm_os_volume_stats(f.database_id, f.file_id)
-FROM	sys.databases
+				) as osVolumes
-CROSS APPLY (
+
-	SELECT	*
+		INSERT INTO @sys_info ( cpu_count, server_memory, sku, engine_edition, hardware_type, total_storage_mb, available_storage_mb, uptime )
-	FROM	@sys_info
+		SELECT	cpu_count,
-) AS sinfo
+				(SELECT total_physical_memory_kb FROM sys.dm_os_sys_memory) AS server_memory,
-OPTION( RECOMPILE );
+				CAST(SERVERPROPERTY('Edition') AS NVARCHAR(64)) as sku,
+				CAST(SERVERPROPERTY('EngineEdition') as smallint) as engine_edition,
+				CASE virtual_machine_type_desc
+					WHEN 'NONE' THEN 'PHYSICAL Machine'
+					ELSE virtual_machine_type_desc
+				END AS hardware_type,
+				@total_disk_size_mb,
+				@available_space_mb,
+				 DATEDIFF(MINUTE,sqlserver_start_time,GETDATE())
+		FROM	sys.dm_os_sys_info
+	END
+END
+	
+SELECT	'sqlserver_server_properties' AS [measurement],
+		REPLACE(@@SERVERNAME,'\',':') AS [sql_instance],
+		s.cpu_count,
+		s.server_memory,
+		s.sku,
+		s.engine_edition,
+		s.hardware_type,
+		s.total_storage_mb,
+		s.available_storage_mb,
+		s.uptime,
+		db_online,
+		db_restoring,
+		db_recovering,
+		db_recoveryPending,
+		db_suspect,
+		db_offline
+FROM	(
+			SELECT	SUM( CASE WHEN state = 0 THEN 1 ELSE 0 END ) AS db_online,
+					SUM( CASE WHEN state = 1 THEN 1 ELSE 0 END ) AS db_restoring,
+					SUM( CASE WHEN state = 2 THEN 1 ELSE 0 END ) AS db_recovering,
+					SUM( CASE WHEN state = 3 THEN 1 ELSE 0 END ) AS db_recoveryPending,
+					SUM( CASE WHEN state = 4 THEN 1 ELSE 0 END ) AS db_suspect,
+					SUM( CASE WHEN state = 10 THEN 1 ELSE 0 END ) AS db_offline
+			FROM	sys.databases
+		) AS dbs
+		CROSS APPLY (
+			SELECT	cpu_count, server_memory, sku, engine_edition, hardware_type, total_storage_mb, available_storage_mb, uptime
+			FROM	@sys_info
+		) AS s
+OPTION( RECOMPILE )
 `
 
 const sqlPerformanceCountersV2 string = `
@@ -416,53 +498,45 @@ SELECT	DISTINCT
 		spi.cntr_type
 FROM	sys.dm_os_performance_counters AS spi
 WHERE	(
-		counter_name IN (
+			counter_name IN (
-			'SQL Compilations/sec',
+				'SQL Compilations/sec',
-			'SQL Re-Compilations/sec',
+				'SQL Re-Compilations/sec',
-			'User Connections',
+				'User Connections',
-			'Batch Requests/sec',
+				'Batch Requests/sec',
-			'Logouts/sec',
+				'Logouts/sec',
-			'Logins/sec',
+				'Logins/sec',
-			'Processes blocked',
+				'Processes blocked',
-			'Latch Waits/sec',
+				'Latch Waits/sec',
-			'Full Scans/sec',
+				'Full Scans/sec',
-			'Index Searches/sec',
+				'Index Searches/sec',
-			'Page Splits/sec',
+				'Page Splits/sec',
-			'Page Lookups/sec',
+				'Page Lookups/sec',
-			'Page Reads/sec',
+				'Page Reads/sec',
-			'Page Writes/sec',
+				'Page Writes/sec',
-			'Readahead Pages/sec',
+				'Readahead Pages/sec',
-			'Lazy Writes/sec',
+				'Lazy Writes/sec',
-			'Checkpoint Pages/sec',
+				'Checkpoint Pages/sec',
-			'Page life expectancy',
+				'Page life expectancy',
-			'Log File(s) Size (KB)',
+				'Log File(s) Size (KB)',
-			'Log File(s) Used Size (KB)',
+				'Log File(s) Used Size (KB)',
-			'Data File(s) Size (KB)',
+				'Data File(s) Size (KB)',
-			'Transactions/sec',
+				'Transactions/sec',
-			'Write Transactions/sec',
+				'Write Transactions/sec',
-			'Active Temp Tables',
+				'Active Temp Tables',
-			'Temp Tables Creation Rate',
+				'Temp Tables Creation Rate',
-			'Temp Tables For Destruction',
+				'Temp Tables For Destruction',
-			'Free Space in tempdb (KB)',
+				'Free Space in tempdb (KB)',
-			'Version Store Size (KB)',
+				'Version Store Size (KB)',
-			'Memory Grants Pending',
+				'Memory Grants Pending',
-			'Free list stalls/sec',
+				'Memory Grants Outstanding',
-			'Buffer cache hit ratio',
+				'Free list stalls/sec',
-			'Buffer cache hit ratio base',
+				'Buffer cache hit ratio',
-			'Backup/Restore Throughput/sec',
+				'Buffer cache hit ratio base',
-			'Total Server Memory (KB)',
+				'Backup/Restore Throughput/sec',
-			'Target Server Memory (KB)',
+				'Total Server Memory (KB)',
-			'Forwarded Recs/sec'
+				'Target Server Memory (KB)',
-		)
-		) OR (
-			instance_name IN ('_Total','Column store object pool')
-			AND counter_name IN (
 				'Log Flushes/sec',
 				'Log Flush Wait Time',
-				'Lock Timeouts/sec',
-				'Number of Deadlocks/sec',
-				'Lock Waits/sec',
-				'Latch Waits/sec',
 				'Memory broker clerk size',
 				'Log Bytes Flushed/sec',
 				'Bytes Sent to Replica/sec',
@@ -477,29 +551,18 @@ WHERE	(
 				'Flow Control/sec',
 				'Resent Messages/sec',
 				'Redone Bytes/sec',
-				'XTP Memory Used (KB)'
+				'XTP Memory Used (KB)',
-			) OR ( 
+				'Transaction Delay',
-				counter_name IN (
+				'Log Bytes Received/sec',
-					'Log Bytes Received/sec',
+				'Log Apply Pending Queue',
-					'Log Apply Pending Queue',
+				'Redone Bytes/sec',
-					'Redone Bytes/sec',
+				'Recovery Queue',
-					'Recovery Queue',
+				'Log Apply Ready Queue',
-					'Log Apply Ready Queue'
-				)
-				AND instance_name = '_Total'
-			)
-		) OR (
-			counter_name IN ('Transaction Delay')
-		) OR (
-			counter_name IN (
 				'CPU usage %',
 				'CPU usage % base',
 				'Queued requests',
 				'Requests completed/sec',
-				'Blocked tasks'
+				'Blocked tasks',
-			)
-		) OR (
-			counter_name IN (
 				'Active memory grant amount (KB)',
 				'Disk Read Bytes/sec',
 				'Disk Read IO Throttled/sec',
@@ -507,11 +570,22 @@ WHERE	(
 				'Disk Write Bytes/sec',
 				'Disk Write IO Throttled/sec',
 				'Disk Write IO/sec',
-				'Used memory (KB)'
+				'Used memory (KB)',
+				'Forwarded Recs/sec',
+				'Background Writer pages/sec',
+				'Percent Log Used'
 			)
-		) OR  (
+		) OR (
 			object_name LIKE '%User Settable%'
 			OR object_name LIKE '%SQL Errors%'
+		) OR (
+			instance_name IN ('_Total')
+			AND counter_name IN (
+				'Lock Timeouts/sec',
+				'Number of Deadlocks/sec',
+				'Lock Waits/sec',
+				'Latch Waits/sec'
+			)
 		)
 
 DECLARE @SQL NVARCHAR(MAX)
@@ -617,6 +691,7 @@ LEFT OUTER JOIN ( VALUES
 ('CMEMPARTITIONED','Memory'),
 ('CMEMTHREAD','Memory'),
 ('CXPACKET','Parallelism'),
+('CXCONSUMER','Parallelism'),
 ('DBMIRROR_DBM_EVENT','Mirroring'),
 ('DBMIRROR_DBM_MUTEX','Mirroring'),
 ('DBMIRROR_EVENTS_QUEUE','Mirroring'),
@@ -1098,10 +1173,8 @@ ws.wait_type NOT IN (
 	N'PARALLEL_REDO_WORKER_WAIT_WORK',
 	N'PREEMPTIVE_HADR_LEASE_MECHANISM', N'PREEMPTIVE_SP_SERVER_DIAGNOSTICS',
 	N'PREEMPTIVE_OS_LIBRARYOPS', N'PREEMPTIVE_OS_COMOPS', N'PREEMPTIVE_OS_CRYPTOPS',
-	N'PREEMPTIVE_OS_PIPEOPS', N'PREEMPTIVE_OS_AUTHENTICATIONOPS',
+	N'PREEMPTIVE_OS_PIPEOPS','PREEMPTIVE_OS_GENERICOPS', N'PREEMPTIVE_OS_VERIFYTRUST',
-	N'PREEMPTIVE_OS_GENERICOPS', N'PREEMPTIVE_OS_VERIFYTRUST',
+	N'PREEMPTIVE_OS_DEVICEOPS', 
-	N'PREEMPTIVE_OS_FILEOPS', N'PREEMPTIVE_OS_DEVICEOPS', N'PREEMPTIVE_OS_QUERYREGISTRY',
-	N'PREEMPTIVE_OS_WRITEFILE',
 	N'PREEMPTIVE_XE_CALLBACKEXECUTE', N'PREEMPTIVE_XE_DISPATCHER',
 	N'PREEMPTIVE_XE_GETTARGETSTATE', N'PREEMPTIVE_XE_SESSIONCOMMIT',
 	N'PREEMPTIVE_XE_TARGETINIT', N'PREEMPTIVE_XE_TARGETFINALIZE',
@@ -1113,14 +1186,15 @@ ws.wait_type NOT IN (
 	N'SLEEP_DCOMSTARTUP', N'SLEEP_MASTERDBREADY', N'SLEEP_MASTERMDREADY',
 	N'SLEEP_MASTERUPGRADED', N'SLEEP_MSDBSTARTUP', N'SLEEP_SYSTEMTASK', N'SLEEP_TASK',
 	N'SLEEP_TEMPDBSTARTUP', N'SNI_HTTP_ACCEPT', N'SP_SERVER_DIAGNOSTICS_SLEEP',
-	N'SQLTRACE_BUFFER_FLUSH', N'SQLTRACE_INCREMENTAL_FLUSH_SLEEP', N'SQLTRACE_WAIT_ENTRIES',
+	N'SQLTRACE_BUFFER_FLUSH', N'SQLTRACE_INCREMENTAL_FLUSH_SLEEP', 
+	N'SQLTRACE_WAIT_ENTRIES',
 	N'WAIT_FOR_RESULTS', N'WAITFOR', N'WAITFOR_TASKSHUTDOWN', N'WAIT_XTP_HOST_WAIT',
 	N'WAIT_XTP_OFFLINE_CKPT_NEW_LOG', N'WAIT_XTP_CKPT_CLOSE',
 	N'XE_BUFFERMGR_ALLPROCESSED_EVENT', N'XE_DISPATCHER_JOIN',
-	N'XE_DISPATCHER_WAIT', N'XE_LIVE_TARGET_TVF', N'XE_TIMER_EVENT')
+	N'XE_DISPATCHER_WAIT', N'XE_LIVE_TARGET_TVF', N'XE_TIMER_EVENT',
+	N'SOS_WORK_DISPATCHER','RESERVED_MEMORY_ALLOCATION_EXT')
 AND waiting_tasks_count > 0
-ORDER BY
+AND wait_time_ms > 100
-waiting_tasks_count DESC
 OPTION (RECOMPILE);
 `