Mysql sql slow query monitoring script code example

#The overall effect is that both the table and the log file are written when the global is turned on, but for general_log only the table is written, and for slow_query_log, both the table and the log file are recorded.
general_log=1#Open the MySQL SQL execution log slow_query_log=1#Open the MySQL slow SQL log #After setting, it will affect general_log and slow_query_log,
log_output=table,File#Log output will be written to the table and the log file. In order to facilitate the program to count, it is best to write to the table#General_log_file is not configured here, so general_log will only write to the table#In mysql5.1.29 and above, set the following to open mysql to record the executed sql in the file#general_log_file=/log/general.log

#5.1.29 and before:
#log=/var/lib/mysql/sql_row.log
long_query_time=1#Set MySQL's slow query to a query that takes more than 1 second slow_query_log_file=/log/slow.log

CREATE TABLE `slow_log_dba` (
 `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 `user_host` mediumtext NOT NULL,
 `query_time` time NOT NULL,
 `lock_time` time NOT NULL,
 `rows_sent` int(11) NOT NULL,
 `rows_examined` int(11) NOT NULL,
 `db` varchar(512) NOT NULL,
 `last_insert_id` int(11) NOT NULL,
 `insert_id` int(11) NOT NULL,
 `server_id` int(10) unsigned NOT NULL,
 `sql_text` mediumtext NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Slow log for dba';

CREATE TABLE `general_log_dba` (
 `event_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 `user_host` mediumtext NOT NULL,
 `thread_id` int(11) NOT NULL,
 `server_id` int(10) unsigned NOT NULL,
 `command_type` varchar(64) NOT NULL,
 `argument` mediumtext NOT NULL,
 KEY `user_host` (`user_host`(200)),
 KEY `event_time` (`event_time`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='general log for dba op';

# -*- coding: utf-8 -*-
__author__ = 'river'
import MySQLdb as mysql
import re
from datetime import datetime, timedelta
import smtplib
from email.mime.text import MIMEText
def sendHtmlMail(mailcontent,myip):
  try:
    yestoday=(datetime.now()-timedelta(days=1)).strftime("%Y-%m-%d")
    sender = '[email protected]'
    receiver = ['[email protected]']
    subject = myip+'mysql operation report '+yestoday
    smtpserver = 'smtp.exmail.xx.com'
    username = '[email protected]'
    password = 'xxxxx'
    msg = MIMEText(mailcontent,'html','utf-8')#'Hello','text','utf-8'
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = '[email protected]'
    smtp = smtplib.SMTP()
    smtp.connect(smtpserver)
    smtp.login(username, password)
    smtp.sendmail(sender, receiver, msg.as_string())
    smtp.quit()
  except Exception, e:
    print e,'send mail error'
if __name__ == '__main__':
  result=None
  htmlfile='mysqlLogMon.html'
  myiplist=['192.168.10.10','192.168.10.19']
  yestoday=(datetime.now()-timedelta(days=1)).strftime("%Y-%m-%d 00:00:00")
  today = datetime.now().strftime("%Y-%m-%d 00:00:00")
  for myip in myiplist:
    sql="select user_host,argument from general_log_dba where event_time >='%s' and event_time <='%s'" %(yestoday,today)
    try:
      dbcon = mysql.connect(host=myip, user='xxxxx', passwd='xxxxx', db='mysql', port=3306, charset='utf8')
      cur = dbcon.cursor()
      print "step 1,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")
      cur.execute(sql)
      result = cur.fetchall()
      cur.close()
      dbcon.close()
    except Exception, e:
      print e,'conn mysql error'
    user_host_set=set()
    print "step 2,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    allhash={}
    if result:
      for user_host,argument in result:
        argument_delcom=re.compile(r'(\/\*(\s|.)*?\*\/)').sub("",argument).strip().replace(u"\x00",'').lower()
        if re.compile(r'^access.*').match(argument_delcom) or re.compile(r'^.*@.*on.*').match(argument_delcom) or re.compile(r'^grant.*').match(argument_delcom):
          tmpargument = argument_delcom.strip()
        else:
          tmpargument=argument_delcom.split(' ')[0].strip()
          if len(tmpargument)>30:
            #Some sql is u'select\n\t\t\t\t\tcount(m.enquirymainid)', you can use print repr(tmpargument)
            tmpargument=argument_delcom.split('\n')[0].strip()
        #If it is all comments, then this item will not be counted if not tmpargument or tmpargument.strip()=='' or tmpargument.strip()==' ':
          continue
        if allhash.has_key(user_host):
          allhash[user_host][tmpargument]=allhash[user_host].get(tmpargument,0)+1
        else:
          allhash[user_host]={tmpargument:1}
      print "step 3,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")
      headhtml='''
      
      '''
      print "step 4,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")
      with open(htmlfile,'w') as htmlfileobj:
        htmlfileobj.write(headhtml)
        htmlfileobj.flush()
      print "step 5,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")
      with open(htmlfile,'a') as htmlfileobj:
        for hostkey in allhash.keys():
          listtmp=sorted(allhash[hostkey].iteritems(),key=lambda labkey:labkey[1],reverse=True)
          rowspan = len(allhash[hostkey])
          #htmlfileobj.write()
          tmpline = '' % (rowspan, hostkey.encode ('utf-8'))
          htmlfileobj.write(tmpline)
          countn=0
          for runsql,count in listtmp:
            if countn==0:
              tmpline='' %(runsql.encode('utf-8'),count)
            else:
              tmpline='' %(runsql.encode('utf-8'),count)
            countn+=1
            htmlfileobj.write(tmpline)
        tmpline='''
The number of times the user executed SQL was %s %s %s
%s %s
'''
        htmlfileobj.write(tmpline)
      with open(htmlfile,'r') as htmlfileobj:
        mailcontent = htmlfileobj.read()
      sendHtmlMail(mailcontent,myip)
    else:
      print 'sql result is None, exit ing'
    print "step 6,"+myip+','+datetime.now().strftime("%Y-%m-%d %H:%M:%S")