From 4d93a433701fadf7e87f08eb553123b02fedf38c Mon Sep 17 00:00:00 2001 From: Christine Date: Mon, 21 Mar 2016 14:44:48 +0100 Subject: [PATCH 01/13] removing deprecated info. --- build/updateCVElist.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/updateCVElist.pl b/build/updateCVElist.pl index c122be9..535d26a 100644 --- a/build/updateCVElist.pl +++ b/build/updateCVElist.pl @@ -21,8 +21,8 @@ sub AUTOLOAD { my $mech = WWW::Mechanize->new(); $mech->agent('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'); -#$mech->proxy( ['http'], 'http://10.236.240.71:3128' ); -#$mech->proxy( ['https'], 'http://10.236.240.71:3128' ); +#$mech->proxy( ['http'], 'http://XXX.XXX.XXX.XXX:3128' ); +#$mech->proxy( ['https'], 'http://XXX.XXX.XXX.XXX:3128' ); $mech->env_proxy; From 216885c30e27293788c77bf1a3cc654ca8ba8d1f Mon Sep 17 00:00:00 2001 From: Christine Date: Mon, 21 Mar 2016 15:51:16 +0100 Subject: [PATCH 02/13] Support user parameter alone Support advanced debug for access denied issue --- mysqltuner.pl | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index e0012b7..d541790 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -505,7 +505,21 @@ sub mysql_setup { $doremote = 1; } } - + # Did we already get a username without password on the command line? + if ( $opt{user} ne 0 and $opt{pass} eq 0 ) { + $mysqllogin = "-u $opt{user} " . $remotestring; + my $loginstatus = `$mysqladmincmd ping $mysqllogin 2>&1`; + if ( $loginstatus =~ /mysqld is alive/ ) { + goodprint + "Logged in using credentials passed on the command line"; + return 1; + } + else { + badprint + "Attempted to use login credentials, but they were invalid"; + exit 1; + } + } # Did we already get a username and password passed on the command line? if ( $opt{user} ne 0 and $opt{pass} ne 0 ) { $mysqllogin = "-u $opt{user} -p\"$opt{pass}\"" . $remotestring; @@ -682,6 +696,15 @@ sub select_array { my $req = shift; debugprint "PERFORM: $req "; my @result = `$mysqlcmd $mysqllogin -Bse "$req" 2>>/dev/null`; + if ($? != 0) { + badprint "failed to execute: $req"; + badprint "FAIL Execute SQL / return code: $?"; + debugprint "CMD : $mysqlcmd"; + debugprint "OPTIONS: $mysqllogin"; + debugprint `$mysqlcmd $mysqllogin -Bse "$req" 2>&1`; + exit $?; + } + debugprint "select_array: return code : $?"; chomp(@result); return @result; } @@ -691,6 +714,15 @@ sub select_one { my $req = shift; debugprint "PERFORM: $req "; my $result = `$mysqlcmd $mysqllogin -Bse "$req" 2>>/dev/null`; + if ($? != 0) { + badprint "failed to execute: $req"; + badprint "FAIL Execute SQL / return code: $?"; + debugprint "CMD : $mysqlcmd"; + debugprint "OPTIONS: $mysqllogin"; + debugprint `$mysqlcmd $mysqllogin -Bse "$req" 2>&1`; + exit $?; + } + debugprint "select_array: return code : $?"; chomp($result); return $result; } @@ -838,10 +870,12 @@ sub security_recommendations { $PASS_COLUMN_NAME='authentication_string'; } debugprint "Password column = $PASS_COLUMN_NAME"; - #exit(0); + # Looking for Anonymous users my @mysqlstatlist = select_array "SELECT CONCAT(user, '\@', host) FROM mysql.user WHERE TRIM(USER) = '' OR USER IS NULL"; + debugprint Dumper \@mysqlstatlist; + #exit 0; if (@mysqlstatlist) { foreach my $line ( sort @mysqlstatlist ) { chomp($line); @@ -1893,7 +1927,7 @@ sub mysql_stats { push( @generalrec, "Upgrade MySQL to version 4+ to utilize query caching" ); } - elsif (mysql_version_ge(5,6)) + elsif (mysql_version_ge(5,5)) { if ( $myvar{'query_cache_type'} ne "OFF" ) { badprint "Query cache should be disabled by default due to mutex contention."; From 42b09f83abe15ed9d8f832d6e2982a4a5a2d436a Mon Sep 17 00:00:00 2001 From: Christine Date: Mon, 21 Mar 2016 17:11:20 +0100 Subject: [PATCH 03/13] Adding system recommandations based on open port trying to detect local vulnerabilities.csv --- mysqltuner.pl | 60 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index d541790..e816a20 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# mysqltuner.pl - Version 1.6.6 +# mysqltuner.pl - Version 1.6.7 # High Performance MySQL Tuning Script # Copyright (C) 2006-2015 Major Hayden - major@mhtx.net # @@ -51,7 +51,7 @@ use Data::Dumper; $Data::Dumper::Pair = " : "; # Set up a few variables for use in the script -my $tunerversion = "1.6.6"; +my $tunerversion = "1.6.7"; my ( @adjvars, @generalrec ); # Set defaults @@ -158,6 +158,7 @@ $basic_password_files = "/usr/share/mysqltuner/basic_passwords.txt" $opt{cvefile} = "/usr/share/mysqltuner/vulnerabilities.csv" unless ( defined $opt{cvefile} and -f "$opt{cvefile}"); $opt{cvefile} ='' unless -f "$opt{cvefile}"; +$opt{cvefile} ='./vulnerabilities.csv' if -f './vulnerabilities.csv'; # my $outputfile = undef; @@ -857,6 +858,58 @@ sub cve_recommendations { } +sub get_opened_ports { + my @opened_ports=`netstat -ltn`; + map { + s/.*:(\d+)\s.*$/$1/; + s/\D//g; + } @opened_ports; + @opened_ports = sort {$a <=> $b} grep { !/^$/ } @opened_ports; + debugprint Dumper \@opened_ports; + return @opened_ports; +} + +sub is_open_port { + my $port=shift; + if ( grep { /^$port$/ } get_opened_ports ) { + return 1; + } + return 0; +} +sub system_recommendations { + prettyprint "\n-------- System Linux Recommendations ---------------------------------------"; + my $os = `uname`; + + unless ($os =~ /Linux/i) { + infoprint "Skipped due to non Linux server"; + return; + } + + prettyprint "Look for related Linux system recommandations"; + + my @opened_ports=get_opened_ports; + infoprint "There is ". scalar @opened_ports. " listening port(s) on this server."; + if (scalar(@opened_ports) > 10) { + badprint "There is too many listening ports: ". scalar(@opened_ports). " > 10"; + push( @generalrec, "Consider dedicating a server for your database installation with less services running on !" ); + } else { + goodprint "There is less than 10 opened ports on this server."; + } + + if ( is_open_port(80) or is_open_port(443) ) { + badprint "There is Apache like server running on 80 or 443 port."; + push( @generalrec, "Consider dedicating a server for Web server in production !" ); + } else { + goodprint "No Web server runing on 80 and 444 port."; + } + if ( is_open_port(8080) or is_open_port(8443) ) { + badprint "There is Application server running on 8080 or 8443 port."; + push( @generalrec, "Consider dedicating a server for Application server in production !" ); + } else { + goodprint "No Application server runing on 8080 or 8443 port."; + } +} + sub security_recommendations { prettyprint "\n-------- Security Recommendations -------------------------------------------"; @@ -3035,6 +3088,7 @@ get_all_vars; # Toss variables/status into hashes get_tuning_info; # Get information about the tuning connexion validate_mysql_version; # Check current MySQL version check_architecture; # Suggest 64-bit upgrade +system_recommendations; # avoid to many service on the same host check_storage_engines; # Show enabled storage engines mysql_databases; # Show informations about databases mysql_indexes; # Show informations about indexes @@ -3067,7 +3121,7 @@ __END__ =head1 NAME - MySQLTuner 1.6.6 - MySQL High Performance Tuning Script + MySQLTuner 1.6.7 - MySQL High Performance Tuning Script =head1 IMPORTANT USAGE GUIDELINES From 1ffee5606a82f93973972355222c002ff1df0fc4 Mon Sep 17 00:00:00 2001 From: Christine Date: Mon, 21 Mar 2016 17:41:14 +0100 Subject: [PATCH 04/13] Adding banned port option to detect banned ports --- mysqltuner.pl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index e816a20..829acec 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -72,7 +72,8 @@ my %opt = ( "skipsize" => 0, "checkversion" => 0, "buffers" => 0, - "passwordfile" => 0, + "passwordfile" => 0, + "bannedports" => '', "outputfile" => 0, "dbstat" => 0, "idxstat" => 0, @@ -92,6 +93,7 @@ GetOptions( 'mysqlcmd=s', 'help', 'buffers', 'skippassword', 'passwordfile=s', 'outputfile=s', 'silent', 'dbstat', 'json', 'idxstat', 'noask', 'template=s', 'reportfile=s', 'cvefile=s', + 'bannedports=s', ); if ( defined $opt{'help'} && $opt{'help'} == 1 ) { usage(); } @@ -134,6 +136,7 @@ sub usage { . " --debug Print debug information\n" . " --dbstat Print database information\n" . " --idxstat Print index information\n" + . " --bannedports ports banned separated by comma(,)\n" . " --cvefile CVE File for vulnerability checks\n" . " --nocolor Don't print output in color\n" . " --json Print result as JSON string\n" @@ -160,6 +163,9 @@ $opt{cvefile} = "/usr/share/mysqltuner/vulnerabilities.csv" $opt{cvefile} ='' unless -f "$opt{cvefile}"; $opt{cvefile} ='./vulnerabilities.csv' if -f './vulnerabilities.csv'; +$opt{'bannedports'}='' unless defined($opt{'bannedports'}); +my @banned_ports=split ',', $opt{'bannedports'}; + # my $outputfile = undef; $outputfile = abs_path( $opt{outputfile} ) unless $opt{outputfile} eq "0"; @@ -908,6 +914,14 @@ sub system_recommendations { } else { goodprint "No Application server runing on 8080 or 8443 port."; } + foreach my $banport (@banned_ports) { + if ( is_open_port($banport) ) { + badprint "Banned port: $banport is opened.."; + push( @generalrec, "Port $banport is opened. Consider stopping program handling this port." ); + } else { + goodprint "$banport is not opened."; + } + } } sub security_recommendations { From 805651d44b25bc37e0f0690b8b09df56ad636e4b Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 09:54:26 +0100 Subject: [PATCH 05/13] Fix bug on index selectivity from mariaDB Adding type in index summary --- mysqltuner.pl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index 829acec..91a8e0b 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -2908,7 +2908,7 @@ FROM INFORMATION_SCHEMA.STATISTICS s , MAX(SEQ_IN_INDEX) AS max_columns FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA NOT IN ('mysql', 'information_schema', 'performance_schema') - AND INDEX_TYPE <> "FULLTEXT" + AND INDEX_TYPE <> 'FULLTEXT' GROUP BY TABLE_SCHEMA, TABLE_NAME, INDEX_NAME ) AS s2 ON s.TABLE_SCHEMA = s2.TABLE_SCHEMA @@ -2933,18 +2933,20 @@ ENDSQL infoprint " +-- NB COLS : " . $info[3] . " column(s)"; infoprint " +-- CARDINALITY : " . $info[4] . " distinct values"; infoprint " +-- NB ROWS : " . $info[5] . " rows"; - infoprint " +-- SELECTIVITY : " . $info[6] . "%"; + infoprint " +-- TYPE : " . $info[6] ; + infoprint " +-- SELECTIVITY : " . $info[7] . "%"; $result{'Indexes'}{ $info[1] }{'Colunm'} = $info[0]; $result{'Indexes'}{ $info[1] }{'Sequence number'} = $info[2]; $result{'Indexes'}{ $info[1] }{'Number of collunm'} = $info[3]; $result{'Indexes'}{ $info[1] }{'Cardianality'} = $info[4]; $result{'Indexes'}{ $info[1] }{'Row number'} = $info[5]; - $result{'Indexes'}{ $info[1] }{'Selectivity'} = $info[6]; - if ( $info[6] < 25 ) { + $result{'Indexes'}{ $info[1] }{'Index Type'} = $info[6]; + $result{'Indexes'}{ $info[1] }{'Selectivity'} = $info[7]; + if ( $info[7] < 25 ) { badprint "$info[1] has a low selectivity"; } - } + } return unless ( defined( $myvar{'performance_schema'} ) @@ -3173,6 +3175,7 @@ You must provide the remote server's total memory when connecting to other serve --debug Print debug information --dbstat Print database information --idxstat Print index information + --bannedports ports banned separated by comma(,) --cvefile CVE File for vulnerability checks --nocolor Don't print output in color --json Print result as JSON string From 200d091362a41537656ae1a9f95af12c76cb3cf7 Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 14:44:01 +0100 Subject: [PATCH 06/13] Adding system recommandations for banned ports and memory used by other process --- mysqltuner.pl | 65 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index 91a8e0b..33ace26 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -826,14 +826,25 @@ sub get_all_vars { } } -sub get_basic_passwords { +sub remove_cr { + map { s/\n$//g; } @_; +} +sub remove_empty { + grep { $_ ne '' } @_; +} +sub get_file_contents { my $file = shift; open( FH, "< $file" ) or die "Can't open $file for read: $!"; my @lines = ; close FH or die "Cannot close $file: $!"; + remove_cr \@lines; return @lines; } +sub get_basic_passwords { + return get_file_contents(shift); +} + sub cve_recommendations { prettyprint "\n-------- CVE Security Recommendations ---------------------------------------"; @@ -882,16 +893,60 @@ sub is_open_port { } return 0; } + +sub get_process_memory { + my $pid=shift; + return 0 unless -f "/proc/$pid/status"; + my @pdata= grep { /RSS:/ } get_file_contents "/proc/$pid/status"; + map { + s/.*RSS:\s*(\d+)\s*kB\s*$/$1*1024/ge + } @pdata; + return $pdata[0]; +} + +sub get_other_process_memory { + my @procs=`ps -eo pid,cmd`; + map { s/.*mysqld.*//; s/.*\[.*\].*//; s/^\s+$//g; s/.*PID.*CMD.*//; s/.*systemd.*//;} @procs; + map {s/\s*?(\d+)\s*.*/$1/g;} @procs; + remove_cr @procs; + @procs=remove_empty @procs; + my $totalMemOther=0; + map { + $totalMemOther+=get_process_memory($_); + } @procs; + return $totalMemOther; +} + +sub get_os_release { + return "Unknown OS release" unless -f "/etc/system-release"; + my @info_release=get_file_contents "/etc/system-release"; + remove_cr @info_release; + return $info_release[0]; +} sub system_recommendations { prettyprint "\n-------- System Linux Recommendations ---------------------------------------"; my $os = `uname`; - unless ($os =~ /Linux/i) { infoprint "Skipped due to non Linux server"; return; } - prettyprint "Look for related Linux system recommandations"; + #prettyprint '-'x78; + infoprint get_os_release; + + my $omem=get_other_process_memory; + infoprint "User process except mysqld used ". hr_bytes_rnd($omem) . " RAM."; + if ( (0.15*$physical_memory) < $omem) { + badprint "Other user process except mysqld used more than 15% of total physical memory ". percentage($omem, $physical_memory). "% (".hr_bytes_rnd($omem). " / ".hr_bytes_rnd($physical_memory).")"; + push( @generalrec, "Consider stopping or dedicate server for additionnal process other than mysqld." ); + push( @adjvars, "DON'T APPLY SETTINGS BECAUSE THERE IS TOO MANY PROCESS RUNNING ON THIS SERVER. OOM KILL CAN OCCURS !" ); + + + } else { + } + + #if ($omem > + #exit 0; my @opened_ports=get_opened_ports; infoprint "There is ". scalar @opened_ports. " listening port(s) on this server."; @@ -1199,6 +1254,7 @@ sub check_architecture { } } $result{'OS'}{'Architecture'} = "$arch bits"; + } # Start up a ton of storage engine counts/statistics @@ -1275,8 +1331,7 @@ sub check_storage_engines { $result{'Databases'}{'List'} = [@dblist]; infoprint "Status: $engines"; if ( mysql_version_ge( 5, 1, 5 ) ) { - -# MySQL 5 servers can have table sizes calculated quickly from information schema + # MySQL 5 servers can have table sizes calculated quickly from information schema my @templist = select_array "SELECT ENGINE,SUM(DATA_LENGTH+INDEX_LENGTH),COUNT(ENGINE),SUM(DATA_LENGTH),SUM(INDEX_LENGTH) FROM information_schema.TABLES WHERE TABLE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'mysql') AND ENGINE IS NOT NULL GROUP BY ENGINE ORDER BY ENGINE ASC;"; From 1a3ce440751ccf6a494152f2c680ec96f15cc861 Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Tue, 22 Mar 2016 14:49:22 +0100 Subject: [PATCH 07/13] Update INTERNALS.md Adding new system recommandations information --- INTERNALS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/INTERNALS.md b/INTERNALS.md index ed4f69d..151666e 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -66,6 +66,10 @@ ## MySQLTuner system checks * 32-bit w/>2GB RAM check +* Check number of opened ports (warning if more than 9 ports opened) +* Check 80, 8080, 443 and 8443 ports if warning are raised if there are opened +* Check if some banned ports are not opened (option --bannedports separated by comma) +* Check if non kernel and user process except mysqld are not using more than 15% of total physical memory) ## MySQLTuner Server version checks * EOL MySQL version check From ffed6a3715422391f41147f8d2b64b729f9f03e5 Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 15:20:18 +0100 Subject: [PATCH 08/13] Adding threadpool information #138 --- mysqltuner.pl | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index 33ace26..771a215 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -785,7 +785,11 @@ sub get_all_vars { if ( ( $myvar{'ignore_builtin_innodb'} || "" ) eq "ON" ) { $myvar{'have_innodb'} = "NO"; } - + + $myvar{'have_threadpool'} = "NO"; + if ( defined ( $myvar{'thread_pool_size'} ) and $myvar{'thread_pool_size'} > 0 ) { + $myvar{'have_threadpool'} = "YES"; + } # have_* for engines is deprecated and will be removed in MySQL 5.6; # check SHOW ENGINES and set corresponding old style variables. # Also works around MySQL bug #59393 wrt. skip-innodb @@ -2503,13 +2507,33 @@ sub mariadb_threadpool { # AriaDB unless ( defined $myvar{'have_threadpool'} - && $myvar{'have_threadpool'} eq "YES" - && defined $enginestats{'Aria'} ) + && $myvar{'have_threadpool'} eq "YES" ) { infoprint "ThreadPool stat is disabled."; return; } - infoprint "ThreadPool stat is enabled."; + infoprint "ThreadPool stat is enabled."; + infoprint "Thread Pool Size: ".$myvar{'thread_pool_size'}. " thread(s)."; + + if ($myvar{'have_innodb'} eq 'YES') { + if ($myvar{'thread_pool_size'}< 16 or $myvar{'thread_pool_size'}>36) { + badprint "thread_pool_size between 16 and 36 when using InnoDB storage engine."; + push( @generalrec, "Thread pool size for InnoDB usage (".$myvar{'thread_pool_size'}.")" ); + push( @adjvars, "thread_pool_size between 16 and 36 for InnoDB usage" ); + } else { + goodprint "thread_pool_size between 16 and 36 when using InnoDB storage engine."; + } + return; + } + if ($myvar{'have_isam'} eq 'YES') { + if ($myvar{'thread_pool_size'}<4 or $myvar{'thread_pool_size'}>8) { + badprint "thread_pool_size between 4 and 8 when using MyIsam storage engine."; + push( @generalrec, "Thread pool size for MyIsam usage (".$myvar{'thread_pool_size'}.")" ); + push( @adjvars, "thread_pool_size between 4 and 8 for MyIsam usage" ); + } else { + goodprint "thread_pool_size between 4 and 8 when using MyISAM storage engine."; + } + } } # Recommendations for Performance Schema @@ -2525,6 +2549,7 @@ sub mysqsl_pfs { infoprint "Performance schema is enabled."; } + # Recommendations for Ariadb sub mariadb_ariadb { prettyprint From a45e83facfa1b3e877923db4267c38080bb08260 Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Tue, 22 Mar 2016 15:23:49 +0100 Subject: [PATCH 09/13] Update INTERNALS.md --- INTERNALS.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/INTERNALS.md b/INTERNALS.md index 151666e..38ac688 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -244,15 +244,5 @@ ## MySQLTuner MariaDB thread pool information -* thread_pool_size -* thread_pool_stall_limit - -* thread_pool_max_threads -* thread_pool_idle_timeout - -* thread_pool_oversubscribe - -* threadpool_threads -* threadpool_idle_threads -* threadpool_threads / thread_pool_size -* threadpool_idle_threads / thread_pool_size +* thread_pool_size between 16 to 36 for Innodb usage +* thread_pool_size between 4 to 8 for MyIsam usage From e6f8f78ffe2bf74e6b2095020e1323500fccb0e6 Mon Sep 17 00:00:00 2001 From: Jean-Marie Renouard Date: Tue, 22 Mar 2016 15:24:24 +0100 Subject: [PATCH 10/13] Update INTERNALS.md --- INTERNALS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INTERNALS.md b/INTERNALS.md index 38ac688..d025858 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -242,7 +242,7 @@ * tokudb_cleaner_iterations * tokudb_fanout -## MySQLTuner MariaDB thread pool information +## MySQLTuner Thread pool information * thread_pool_size between 16 to 36 for Innodb usage * thread_pool_size between 4 to 8 for MyIsam usage From 47ffb270d9c4ddee353b4e667e5d17b18a8342d8 Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 15:35:00 +0100 Subject: [PATCH 11/13] Typo fix --- mysqltuner.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index 771a215..f39b532 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -943,7 +943,7 @@ sub system_recommendations { if ( (0.15*$physical_memory) < $omem) { badprint "Other user process except mysqld used more than 15% of total physical memory ". percentage($omem, $physical_memory). "% (".hr_bytes_rnd($omem). " / ".hr_bytes_rnd($physical_memory).")"; push( @generalrec, "Consider stopping or dedicate server for additionnal process other than mysqld." ); - push( @adjvars, "DON'T APPLY SETTINGS BECAUSE THERE IS TOO MANY PROCESS RUNNING ON THIS SERVER. OOM KILL CAN OCCURS !" ); + push( @adjvars, "DON'T APPLY SETTINGS BECAUSE THERE IS TOO MANY PROCESSES RUNNING ON THIS SERVER. OOM KILL CAN OCCURS !" ); } else { From fb53fa4efce4ead40eb0186e4110462f22936bfd Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 16:36:04 +0100 Subject: [PATCH 12/13] Adding extra information on collation in dbstat --- mysqltuner.pl | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/mysqltuner.pl b/mysqltuner.pl index f39b532..0974d32 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -2883,19 +2883,30 @@ sub mysql_databases { infoprint "There is " . scalar(@dblist) . " Database(s)."; my @totaldbinfo = split /\s/, select_one( -"SELECT SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH) FROM information_schema.TABLES;" +"SELECT SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)) FROM information_schema.TABLES;" ); infoprint "All Databases:"; - infoprint " +-- ROWS : " + infoprint " +-- TABLE : " + . ( $totaldbinfo[4] eq 'NULL' ? 0 : $totaldbinfo[4] ) . ""; + infoprint " +-- ROWS : " . ( $totaldbinfo[0] eq 'NULL' ? 0 : $totaldbinfo[0] ) . ""; - infoprint " +-- DATA : " + infoprint " +-- DATA : " . hr_bytes( $totaldbinfo[1] ) . "(" . percentage( $totaldbinfo[1], $totaldbinfo[3] ) . "%)"; - infoprint " +-- INDEX: " + infoprint " +-- INDEX : " . hr_bytes( $totaldbinfo[2] ) . "(" . percentage( $totaldbinfo[2], $totaldbinfo[3] ) . "%)"; - infoprint " +-- SIZE : " . hr_bytes( $totaldbinfo[3] ) . ""; - + infoprint " +-- SIZE : " . hr_bytes( $totaldbinfo[3] ) . ""; + infoprint " +-- COLLA : " + . ( $totaldbinfo[5] eq 'NULL' ? 0 : $totaldbinfo[5] ) . " (". (join ", ", select_array ("SELECT DISTINCT(TABLE_COLLATION) FROM information_schema.TABLES;")) .")"; + + + if ($totaldbinfo[5]>1) { + badprint $totaldbinfo[5]. " differents collations for tables detected."; + push(@generalrec, "Check your general collation and your database table location are identical."); + } else { + goodprint $totaldbinfo[5]. " collation for tables detected."; + } badprint "Index size is larger than data size \n" if $totaldbinfo[1] < $totaldbinfo[2]; @@ -2907,8 +2918,8 @@ sub mysql_databases { $result{'Databases'}{'All databases'}{'Index Size'} = $totaldbinfo[2]; $result{'Databases'}{'All databases'}{'Index Pct'} = percentage( $totaldbinfo[2], $totaldbinfo[3] ) . "%"; - $result{'Databases'}{'All databases'}{'Total Size'} = $totaldbinfo[3]; - + $result{'Databases'}{'All databases'}{'Total Size'} = $totaldbinfo[3]; + print "\n"; foreach (@dblist) { chomp($_); if ( $_ eq "information_schema" @@ -2921,10 +2932,15 @@ sub mysql_databases { my @dbinfo = split /\s/, select_one( -"SELECT TABLE_SCHEMA, SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(DISTINCT ENGINE) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_' GROUP BY TABLE_SCHEMA ORDER BY TABLE_SCHEMA" +"SELECT TABLE_SCHEMA, SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(DISTINCT ENGINE),COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_' GROUP BY TABLE_SCHEMA ORDER BY TABLE_SCHEMA" ); next unless defined $dbinfo[0]; infoprint "Database: " . $dbinfo[0] . ""; + infoprint " +-- TABLE: " + . ( !defined( $dbinfo[6] ) or $dbinfo[6] eq 'NULL' ? 0 : $dbinfo[6] ) + . ""; + infoprint " +-- COLL : " + . ( $dbinfo[7] eq 'NULL' ? 0 : $dbinfo[7] ) . " (". (join ", ", select_array ("SELECT DISTINCT(TABLE_COLLATION) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_';")) .")"; infoprint " +-- ROWS : " . ( !defined( $dbinfo[1] ) or $dbinfo[1] eq 'NULL' ? 0 : $dbinfo[1] ) . ""; @@ -2940,6 +2956,8 @@ sub mysql_databases { badprint "There are " . $dbinfo[5] . " storage engines. Be careful. \n" if $dbinfo[5] > 1; $result{'Databases'}{ $dbinfo[0] }{'Rows'} = $dbinfo[1]; + $result{'Databases'}{ $dbinfo[0] }{'Tables'} = $dbinfo[6]; + $result{'Databases'}{ $dbinfo[0] }{'Collations'} = $dbinfo[7]; $result{'Databases'}{ $dbinfo[0] }{'Data Size'} = $dbinfo[2]; $result{'Databases'}{ $dbinfo[0] }{'Data Pct'} = percentage( $dbinfo[2], $dbinfo[4] ) . "%"; @@ -2947,6 +2965,12 @@ sub mysql_databases { $result{'Databases'}{ $dbinfo[0] }{'Index Pct'} = percentage( $dbinfo[3], $dbinfo[4] ) . "%"; $result{'Databases'}{ $dbinfo[0] }{'Total Size'} = $dbinfo[4]; + if ($dbinfo[7]>1) { + badprint $dbinfo[7]. " differents collations for database ".$dbinfo[0]; + push(@generalrec, "Check all table collations are identical for all tables in ".$dbinfo[0]. " database."); + } else { + goodprint $dbinfo[7]. " collation for ".$dbinfo[0]. " database."; + } } } From 18941ae560aff89090f7037cfed4e631a91f5820 Mon Sep 17 00:00:00 2001 From: Christine Date: Tue, 22 Mar 2016 17:56:08 +0100 Subject: [PATCH 13/13] Advanced dbstat for collation and charset --- INTERNALS.md | 6 ++++++ mysqltuner.pl | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/INTERNALS.md b/INTERNALS.md index d025858..d24b3c4 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -101,12 +101,18 @@ ## MySQLTuner database information * Per database information + * Tables number * Rows number * Total size * Data size * Percentage of data size * Index size * Percentage of index size + * Collation number + * Check that there is only one collation for all table in a database + * Check that there is only one collation for ll table columns in a database + * Check that there is only one storage engine per user database + ## MySQLTuner index information diff --git a/mysqltuner.pl b/mysqltuner.pl index 0974d32..755426f 100755 --- a/mysqltuner.pl +++ b/mysqltuner.pl @@ -2883,7 +2883,7 @@ sub mysql_databases { infoprint "There is " . scalar(@dblist) . " Database(s)."; my @totaldbinfo = split /\s/, select_one( -"SELECT SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)) FROM information_schema.TABLES;" +"SELECT SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)),COUNT(DISTINCT(ENGINE)) FROM information_schema.TABLES;" ); infoprint "All Databases:"; infoprint " +-- TABLE : " @@ -2899,17 +2899,10 @@ sub mysql_databases { infoprint " +-- SIZE : " . hr_bytes( $totaldbinfo[3] ) . ""; infoprint " +-- COLLA : " . ( $totaldbinfo[5] eq 'NULL' ? 0 : $totaldbinfo[5] ) . " (". (join ", ", select_array ("SELECT DISTINCT(TABLE_COLLATION) FROM information_schema.TABLES;")) .")"; + infoprint " +-- ENGIN : " + . ( $totaldbinfo[6] eq 'NULL' ? 0 : $totaldbinfo[6] ) . " (". (join ", ", select_array ("SELECT DISTINCT(ENGINE) FROM information_schema.TABLES;")) .")"; - if ($totaldbinfo[5]>1) { - badprint $totaldbinfo[5]. " differents collations for tables detected."; - push(@generalrec, "Check your general collation and your database table location are identical."); - } else { - goodprint $totaldbinfo[5]. " collation for tables detected."; - } - badprint "Index size is larger than data size \n" - if $totaldbinfo[1] < $totaldbinfo[2]; - $result{'Databases'}{'All databases'}{'Rows'} = ( $totaldbinfo[0] eq 'NULL' ? 0 : $totaldbinfo[0] ); $result{'Databases'}{'All databases'}{'Data Size'} = $totaldbinfo[1]; @@ -2924,7 +2917,7 @@ sub mysql_databases { chomp($_); if ( $_ eq "information_schema" or $_ eq "performance_schema" - or $_ eq "mysql" + # or $_ eq "mysql" or $_ eq "" ) { next; @@ -2932,7 +2925,7 @@ sub mysql_databases { my @dbinfo = split /\s/, select_one( -"SELECT TABLE_SCHEMA, SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(DISTINCT ENGINE),COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_' GROUP BY TABLE_SCHEMA ORDER BY TABLE_SCHEMA" +"SELECT TABLE_SCHEMA, SUM(TABLE_ROWS), SUM(DATA_LENGTH), SUM(INDEX_LENGTH) , SUM(DATA_LENGTH+INDEX_LENGTH), COUNT(DISTINCT ENGINE),COUNT(TABLE_NAME),COUNT(DISTINCT(TABLE_COLLATION)),COUNT(DISTINCT(ENGINE)) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_' GROUP BY TABLE_SCHEMA ORDER BY TABLE_SCHEMA" ); next unless defined $dbinfo[0]; infoprint "Database: " . $dbinfo[0] . ""; @@ -2951,6 +2944,8 @@ sub mysql_databases { . hr_bytes( $dbinfo[3] ) . "(" . percentage( $dbinfo[3], $dbinfo[4] ) . "%)"; infoprint " +-- TOTAL: " . hr_bytes( $dbinfo[4] ) . ""; + infoprint " +-- ENGIN : " + . ( $dbinfo[8] eq 'NULL' ? 0 : $dbinfo[8] ) . " (". (join ", ", select_array ("SELECT DISTINCT(ENGINE) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$_'")) .")"; badprint "Index size is larger than data size for $dbinfo[0] \n" if $dbinfo[2] < $dbinfo[3]; badprint "There are " . $dbinfo[5] . " storage engines. Be careful. \n" @@ -2971,7 +2966,32 @@ sub mysql_databases { } else { goodprint $dbinfo[7]. " collation for ".$dbinfo[0]. " database."; } + if ($dbinfo[8]>1) { + badprint $dbinfo[8]. " differents engines for database ".$dbinfo[0]; + push(@generalrec, "Check all table engines are identical for all tables in ".$dbinfo[0]. " database."); + } else { + goodprint $dbinfo[8]. " engine for ".$dbinfo[0]. " database."; } + + my @distinct_column_charset=select_array("select DISTINCT(CHARACTER_SET_NAME) from information_schema.COLUMNS where CHARACTER_SET_NAME IS NOT NULL AND TABLE_SCHEMA ='$_'"); + infoprint "Charsets for $dbinfo[0] database table column: ". join (', ', @distinct_column_charset); + if (scalar (@distinct_column_charset)>1 ) { + badprint $dbinfo[0]. " table column(s) has several charsets defined for all text like column(s)."; + push(@generalrec, "Limit charset for column to one charset if possible for ".$dbinfo[0]." database."); + } else { + goodprint $dbinfo[0]. " table column(s) has same charset defined for all text like column(s)."; + } + + my @distinct_column_collation=select_array("select DISTINCT(COLLATION_NAME) from information_schema.COLUMNS where COLLATION_NAME IS NOT NULL AND TABLE_SCHEMA ='$_'"); + infoprint "Collations for $dbinfo[0] database table column: ". join (', ', @distinct_column_collation); + if (scalar (@distinct_column_collation)>1 ) { + badprint $dbinfo[0]. " table column(s) has several collations defined for all text like column(s)."; + push(@generalrec, "Limit collations for column to one collation if possible for ".$dbinfo[0]." database."); + } else { + goodprint $dbinfo[0]. " table column(s) has same collation defined for all text like column(s)."; + } + } + } # Recommendations for Indexes metrics