Merged revisons 3687-3748 from 3.0.5 stable branch.

tbrehm

2013-01-03 10b4c85a7686da07bddb5779f8ceea48acdf412e

 server/cron_daily.php

@@ -39,18 +39,22 @@


// Load required base-classes
$app->uses('ini_parser,file,services,getconf');
$app->uses('ini_parser,file,services,getconf,system');


#######################################################################################################
// store the mailbox statistics in the database
#######################################################################################################

$parse_mail_log = false;
$sql = "SELECT mailuser_id,maildir FROM mail_user WHERE server_id = ".$conf['server_id'];
$records = $app->db->queryAllRecords($sql);
if(count($records) > 0) $parse_mail_log = true;

foreach($records as $rec) {
   if(@is_file($rec['maildir'].'/ispconfig_mailsize')) {

        $parse_mail_log = false;
        
      // rename file
      rename($rec['maildir'].'/ispconfig_mailsize',$rec['maildir'].'/ispconfig_mailsize_save');

@@ -78,10 +82,165 @@
         $sql = "INSERT INTO mail_traffic (month,mailuser_id,traffic) VALUES ('$tstamp',".$rec['mailuser_id'].",$mail_traffic)";
      }
      $app->dbmaster->query($sql);
      echo $sql;
      //echo $sql;

   }

}

if($parse_mail_log == true) {
    $mailbox_traffic = array();
    $mail_boxes = array();
    $mail_rewrites = array(); // we need to read all mail aliases and forwards because the address in amavis is not always the mailbox address
    
    function parse_mail_log_line($line) {
        //Oct 31 17:35:48 mx01 amavis[32014]: (32014-05) Passed CLEAN, [IPv6:xxxxx] [IPv6:xxxxx] <xxx@yyyy> -> <aaaa@bbbb>, Message-ID: <xxxx@yyyyy>, mail_id: xxxxxx, Hits: -1.89, size: 1591, queued_as: xxxxxxx, 946 ms
        
        if(preg_match('/^(\w+\s+\d+\s+\d+:\d+:\d+)\s+[^ ]+\s+amavis.* <([^>]+)>\s+->\s+((<[^>]+>,)+) .*Message-ID:\s+<([^>]+)>.* size:\s+(\d+),.*$/', $line, $matches) == false) return false;
        
        $timestamp = strtotime($matches[1]);
        if(!$timestamp) return false;
        
        $to = array();
        $recipients = explode(',', $matches[3]);
        foreach($recipients as $recipient) {
            $recipient = substr($recipient, 1, -1);
            if(!$recipient || $recipient == $matches[2]) continue;
            $to[] = $recipient;
        }
        
        return array('line' => $line, 'timestamp' => $timestamp, 'size' => $matches[6], 'from' => $matches[2], 'to' => $to, 'message-id' => $matches[5]);
    }

    function add_mailbox_traffic(&$traffic_array, $address, $traffic) {
        global $mail_boxes, $mail_rewrites;
        
        $address = strtolower($address);
        
        if(in_array($address, $mail_boxes) == true) {
            if(!isset($traffic_array[$address])) $traffic_array[$address] = 0;
            $traffic_array[$address] += $traffic;
        } elseif(array_key_exists($address, $mail_rewrites)) {
            foreach($mail_rewrites[$address] as $address) {
                if(!isset($traffic_array[$address])) $traffic_array[$address] = 0;
                $traffic_array[$address] += $traffic;
            }
        } else {
            // this is not a local address - skip it
        }
    }

    $sql = "SELECT email FROM mail_user WHERE server_id = ".$conf['server_id'];
    $records = $app->db->queryAllRecords($sql);
    foreach($records as $record) {
        $mail_boxes[] = $record['email'];
    }
    $sql = "SELECT source, destination FROM mail_forwarding WHERE server_id = ".$conf['server_id'];
    $records = $app->db->queryAllRecords($sql);
    foreach($records as $record) {
        $targets = preg_split('/[\n,]+/', $record['destination']);
        foreach($targets as $target) {
            if(in_array($target, $mail_boxes)) {
                if(isset($mail_rewrites[$record['source']])) $mail_rewrites[$record['source']][] = $target;
                else $mail_rewrites[$record['source']] = array($target);
            }
        }
    }
    
    $state_file = dirname(__FILE__) . '/mail_log_parser.state';
    $prev_line = false;
    $last_line = false;
    $cur_line = false;
    
    if(file_exists($state_file)) {
        $prev_line = parse_mail_log_line(trim(file_get_contents($state_file)));
        //if($prev_line) echo "continuing from previous run, log position: " . $prev_line['message-id'] . " at " . strftime('%d.%m.%Y %H:%M:%S', $prev_line['timestamp']) . "\n";
    }
    
    if(file_exists('/var/log/mail.log')) {
        $fp = fopen('/var/log/mail.log', 'r');
        //echo "Parsing mail.log...\n";
        $l = 0;
        while($line = fgets($fp, 8192)) {
            $l++;
            //if($l % 1000 == 0) echo "\rline $l";
            $cur_line = parse_mail_log_line($line);
            if(!$cur_line) continue;
            
            if($prev_line) {
                // check if this line has to be processed
                if($cur_line['timestamp'] < $prev_line['timestamp']) {
                    $parse_mail_log = false; // we do not need to parse the second file!
                    continue; // already processed
                } elseif($cur_line['timestamp'] == $prev_line['timestamp'] && $cur_line['message-id'] == $prev_line['message-id']) {
                    $parse_mail_log = false; // we do not need to parse the second file!
                    $prev_line = false; // this line has already been processed but the next one has to be!
                    continue;
                }
            }
            
            add_mailbox_traffic($mailbox_traffic, $cur_line['from'], $cur_line['size']);
            foreach($cur_line['to'] as $to) {
                add_mailbox_traffic($mailbox_traffic, $to, $cur_line['size']);
            }
            $last_line = $line; // store for the state file
        }
        fclose($fp);
        //echo "\n";
    }
    
    if($parse_mail_log == true && file_exists('/var/log/mail.log.1')) {
        $fp = fopen('/var/log/mail.log.1', 'r');
        //echo "Parsing mail.log.1...\n";
        $l = 0;
        while($line = fgets($fp, 8192)) {
            $l++;
            //if($l % 1000 == 0) echo "\rline $l";
            $cur_line = parse_mail_log_line($line);
            if(!$cur_line) continue;
            
            if($prev_line) {
                // check if this line has to be processed
                if($cur_line['timestamp'] < $prev_line['timestamp']) continue; // already processed
                if($cur_line['timestamp'] == $prev_line['timestamp'] && $cur_line['message-id'] == $prev_line['message-id']) {
                    $prev_line = false; // this line has already been processed but the next one has to be!
                    continue;
                }
            }
            
            add_mailbox_traffic($mailbox_traffic, $cur_line['from'], $cur_line['size']);
            foreach($cur_line['to'] as $to) {
                add_mailbox_traffic($mailbox_traffic, $to, $cur_line['size']);
            }
        }
        fclose($fp);
        //echo "\n";
    }
    unset($mail_rewrites);
    unset($mail_boxes);
    
    // Save the traffic stats in the sql database
    $tstamp = date('Y-m');
    $sql = "SELECT mailuser_id,email FROM mail_user WHERE server_id = ".$conf['server_id'];
    $records = $app->db->queryAllRecords($sql);
    foreach($records as $rec) {
        if(array_key_exists($rec['email'], $mailbox_traffic)) {
            $sql = "SELECT * FROM mail_traffic WHERE month = '$tstamp' AND mailuser_id = ".$rec['mailuser_id'];
            $tr = $app->dbmaster->queryOneRecord($sql);

            $mail_traffic = $tr['traffic'] + $mailbox_traffic[$rec['email']];
            if($tr['traffic_id'] > 0) {
                $sql = "UPDATE mail_traffic SET traffic = $mail_traffic WHERE traffic_id = ".$tr['traffic_id'];
            } else {
                $sql = "INSERT INTO mail_traffic (month,mailuser_id,traffic) VALUES ('$tstamp',".$rec['mailuser_id'].",$mail_traffic)";
            }
            $app->dbmaster->query($sql);
            //echo $sql;
        }
    }
    
    unset($mailbox_traffic);
    if($last_line) file_put_contents($state_file, $last_line);
}

#######################################################################################################
@@ -113,7 +272,7 @@
}


$sql = "SELECT domain_id, domain, document_root FROM web_domain WHERE stats_type = 'webalizer' AND server_id = ".$conf['server_id'];
$sql = "SELECT domain_id, domain, document_root, web_folder, type FROM web_domain WHERE (type = 'vhost' or type = 'vhostsubdomain') and stats_type = 'webalizer' AND server_id = ".$conf['server_id'];
$records = $app->db->queryAllRecords($sql);

foreach($records as $rec) {
@@ -128,7 +287,7 @@
   }

   $domain = escapeshellcmd($rec['domain']);
   $statsdir = escapeshellcmd($rec['document_root'].'/web/stats');
   $statsdir = escapeshellcmd($rec['document_root'].'/'.($rec['type'] == 'vhostsubdomain' ? $rec['web_folder'] : 'web').'/stats');
   $webalizer = '/usr/bin/webalizer';
   $webalizer_conf_main = '/etc/webalizer/webalizer.conf';
   $webalizer_conf = escapeshellcmd($rec['document_root'].'/log/webalizer.conf');
@@ -154,7 +313,7 @@
// Create awstats statistics
#######################################################################################################

$sql = "SELECT domain_id, domain, document_root, system_user, system_group FROM web_domain WHERE stats_type = 'awstats' AND server_id = ".$conf['server_id'];
$sql = "SELECT domain_id, domain, document_root, web_folder, type, system_user, system_group FROM web_domain WHERE (type = 'vhost' or type = 'vhostsubdomain') and stats_type = 'awstats' AND server_id = ".$conf['server_id'];
$records = $app->db->queryAllRecords($sql);

$web_config = $app->getconf->get_server_config($conf['server_id'], 'web');
@@ -169,9 +328,9 @@
         continue;
      }
   }

    $web_folder = ($rec['type'] == 'vhostsubdomain' ? $rec['web_folder'] : 'web');
   $domain = escapeshellcmd($rec['domain']);
   $statsdir = escapeshellcmd($rec['document_root'].'/web/stats');
   $statsdir = escapeshellcmd($rec['document_root'].'/'.$web_folder.'/stats');
   $awstats_pl = $web_config['awstats_pl'];
   $awstats_buildstaticpages_pl = $web_config['awstats_buildstaticpages_pl'];

@@ -213,7 +372,7 @@
      }
   }

   // awstats_buildstaticpages.pl -update -config=mydomain.com -lang=en -dir=/var/www/domain.com/web/stats -awstatsprog=/path/to/awstats.pl
   // awstats_buildstaticpages.pl -update -config=mydomain.com -lang=en -dir=/var/www/domain.com/'.$web_folder.'/stats -awstatsprog=/path/to/awstats.pl
   // $command = "$awstats_buildstaticpages_pl -update -config='$domain' -lang=".$conf['language']." -dir='$statsdir' -awstatsprog='$awstats_pl'";

   $command = "$awstats_buildstaticpages_pl -month='$awmonth' -year='$awyear' -update -config='$domain' -lang=".$conf['language']." -dir='$statsdir' -awstatsprog='$awstats_pl'";
@@ -236,18 +395,24 @@

   if($awstats_pl != '' && $awstats_buildstaticpages_pl != '' && fileowner($awstats_pl) == 0 && fileowner($awstats_buildstaticpages_pl) == 0) {
      exec($command);
      if(is_file($rec['document_root'].'/web/stats/index.html')) unlink($rec['document_root'].'/web/stats/index.html');
      rename($rec['document_root'].'/web/stats/awstats.'.$domain.'.html',$rec['document_root'].'/web/stats/awsindex.html');
      if(!is_file($rec['document_root']."/web/stats/index.php")) copy("/usr/local/ispconfig/server/conf/awstats_index.php.master",$rec['document_root']."/web/stats/index.php");
      if(is_file($rec['document_root'].'/'.$web_folder.'/stats/index.html')) unlink($rec['document_root'].'/'.$web_folder.'/stats/index.html');
      rename($rec['document_root'].'/'.$web_folder.'/stats/awstats.'.$domain.'.html',$rec['document_root'].'/'.$web_folder.'/stats/awsindex.html');
      if(!is_file($rec['document_root']."/".$web_folder."/stats/index.php")) {
         if(file_exists("/usr/local/ispconfig/server/conf-custom/awstats_index.php.master")) {
            copy("/usr/local/ispconfig/server/conf-custom/awstats_index.php.master",$rec['document_root']."/".$web_folder."/stats/index.php");
         } else {
            copy("/usr/local/ispconfig/server/conf/awstats_index.php.master",$rec['document_root']."/".$web_folder."/stats/index.php");
         }
      }

      $app->log('Created awstats statistics with command: '.$command,LOGLEVEL_DEBUG);
   } else {
      $app->log("No awstats statistics created. Either $awstats_pl or $awstats_buildstaticpages_pl is not owned by root user.",LOGLEVEL_WARN);
   }

   if(is_file($rec['document_root']."/web/stats/index.php")) {
      chown($rec['document_root']."/web/stats/index.php",$rec['system_user']);
      chgrp($rec['document_root']."/web/stats/index.php",$rec['system_group']);
   if(is_file($rec['document_root']."/".$web_folder."/stats/index.php")) {
      chown($rec['document_root']."/".$web_folder."/stats/index.php",$rec['system_user']);
      chgrp($rec['document_root']."/".$web_folder."/stats/index.php",$rec['system_group']);
   }

}
@@ -263,7 +428,7 @@
// Manage and compress web logfiles and create traffic statistics
#######################################################################################################

$sql = "SELECT domain_id, domain, document_root FROM web_domain WHERE server_id = ".$conf['server_id'];
$sql = "SELECT domain_id, domain, document_root FROM web_domain WHERE (type = 'vhost' or type = 'vhostsubdomain') AND server_id = ".$conf['server_id'];
$records = $app->db->queryAllRecords($sql);
foreach($records as $rec) {

@@ -452,7 +617,7 @@
   $current_month = date('Y-m');

   //* Check website traffic quota
   $sql = "SELECT sys_groupid,domain_id,domain,traffic_quota,traffic_quota_lock FROM web_domain WHERE (traffic_quota > 0 or traffic_quota_lock = 'y') and type = 'vhost'";
   $sql = "SELECT sys_groupid,domain_id,domain,traffic_quota,traffic_quota_lock FROM web_domain WHERE (traffic_quota > 0 or traffic_quota_lock = 'y') and (type = 'vhost' OR type = 'vhostsubdomain')";
   $records = $app->db->queryAllRecords($sql);
   if(is_array($records)) {
      foreach($records as $rec) {
@@ -472,7 +637,7 @@

         //* get the traffic
         $tmp = $app->db->queryOneRecord("SELECT SUM(traffic_bytes) As total_traffic_bytes FROM web_traffic WHERE traffic_date like '$current_month%' AND hostname = '$domain'");
         $web_traffic = (int)$tmp['total_traffic_bytes']/1024/1024;
         $web_traffic = round($tmp['total_traffic_bytes']/1024/1024);

         //* Website is over quota, we will disable it
         /*if( ($web_traffic_quota > 0 && $web_traffic > $web_traffic_quota) ||
@@ -521,7 +686,7 @@
               }
               
               //* Send email to client
               if($web_config['overtraffic_notify_admin'] == 'y') {
               if($web_config['overtraffic_notify_client'] == 'y') {
                  $client_group_id = $rec["sys_groupid"];
                  $client = $app->db->queryOneRecord("SELECT client.email FROM sys_group, client WHERE sys_group.client_id = client.client_id and sys_group.groupid = $client_group_id");
                  if($client['email'] != '') {
@@ -592,7 +757,7 @@
      chmod(escapeshellcmd($backup_dir), $backup_dir_permissions);
   }

   $sql = "SELECT * FROM web_domain WHERE server_id = '".$conf['server_id']."' AND type = 'vhost' AND backup_interval != 'none'";
   $sql = "SELECT * FROM web_domain WHERE server_id = '".$conf['server_id']."' AND (type = 'vhost' OR type = 'vhostsubdomain') AND backup_interval != 'none'";
   $records = $app->db->queryAllRecords($sql);
   if(is_array($records)) {
      foreach($records as $rec) {
@@ -629,9 +794,12 @@
            chmod($web_backup_dir.'/'.$web_backup_file, 0750);

            //* Insert web backup record in database
            $insert_data = "(server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",".$web_id.",'web','".$backup_mode."',".time().",'".$app->db->quote($web_backup_file)."')";
            $app->dbmaster->datalogInsert('web_backup', $insert_data, 'backup_id');

            //$insert_data = "(server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",".$web_id.",'web','".$backup_mode."',".time().",'".$app->db->quote($web_backup_file)."')";
            //$app->dbmaster->datalogInsert('web_backup', $insert_data, 'backup_id');
            $sql = "INSERT INTO web_backup (server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",".$web_id.",'web','".$backup_mode."',".time().",'".$app->db->quote($web_backup_file)."')";
            $app->db->query($sql);
            if($app->db->dbHost != $app->dbmaster->dbHost) $app->dbmaster->query($sql);
				
            //* Remove old backups
            $backup_copies = intval($rec['backup_copies']);

@@ -649,9 +817,13 @@
            for ($n = $backup_copies; $n <= 10; $n++) {
               if(isset($files[$n]) && is_file($web_backup_dir.'/'.$files[$n])) {
                  unlink($web_backup_dir.'/'.$files[$n]);
                  $sql = "SELECT backup_id FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($files[$n])."'";
                  $tmp = $app->dbmaster->queryOneRecord($sql);
                  $app->dbmaster->datalogDelete('web_backup', 'backup_id', $tmp['backup_id']);
                  //$sql = "SELECT backup_id FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($files[$n])."'";
                  //$tmp = $app->dbmaster->queryOneRecord($sql);
                  //$app->dbmaster->datalogDelete('web_backup', 'backup_id', $tmp['backup_id']);
                  //$sql = "DELETE FROM web_backup WHERE backup_id = ".intval($tmp['backup_id']);
                  $sql = "DELETE FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($files[$n])."'";
                  $app->db->query($sql);
                  if($app->db->dbHost != $app->dbmaster->dbHost) $app->dbmaster->query($sql);
               }
            }

@@ -659,6 +831,9 @@
            unset($dir_handle);

            //* Remove backupdir symlink and create as directory instead
            $app->uses('system');
            $app->system->web_folder_protection($web_path,false);
				
            if(is_link($web_path.'/backup')) {
               unlink($web_path.'/backup');
            }
@@ -667,6 +842,8 @@
               chown($web_path.'/backup', $rec['system_user']);
               chgrp($web_path.'/backup', $rec['system_group']);
            }
				
            $app->system->web_folder_protection($web_path,true);

         }

@@ -696,10 +873,10 @@

            $web_id = $rec['parent_domain_id'];
            $db_backup_dir = $backup_dir.'/web'.$web_id;
            if(!is_dir($web_backup_dir)) mkdir($web_backup_dir, 0750);
            chmod($web_backup_dir, 0750);
            chown($web_backup_dir, 'root');
            chgrp($web_backup_dir, 'root');
            if(!is_dir($db_backup_dir)) mkdir($db_backup_dir, 0750);
            chmod($db_backup_dir, 0750);
            chown($db_backup_dir, 'root');
            chgrp($db_backup_dir, 'root');

            //* Do the mysql database backup with mysqldump
            $db_id = $rec['database_id'];
@@ -715,8 +892,11 @@
            chgrp($db_backup_dir.'/'.$db_backup_file.'.gz', filegroup($db_backup_dir));

            //* Insert web backup record in database
            $insert_data = "(server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",$web_id,'mysql','sqlgz',".time().",'".$app->db->quote($db_backup_file).".gz')";
            $app->dbmaster->datalogInsert('web_backup', $insert_data, 'backup_id');
            //$insert_data = "(server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",$web_id,'mysql','sqlgz',".time().",'".$app->db->quote($db_backup_file).".gz')";
            //$app->dbmaster->datalogInsert('web_backup', $insert_data, 'backup_id');
            $sql = "INSERT INTO web_backup (server_id,parent_domain_id,backup_type,backup_mode,tstamp,filename) VALUES (".$conf['server_id'].",$web_id,'mysql','sqlgz',".time().",'".$app->db->quote($db_backup_file).".gz')";
            $app->db->query($sql);
            if($app->db->dbHost != $app->dbmaster->dbHost) $app->dbmaster->query($sql);

            //* Remove the uncompressed file
            unlink($db_backup_dir.'/'.$db_backup_file);
@@ -727,22 +907,28 @@
            $dir_handle = dir($db_backup_dir);
            $files = array();
            while (false !== ($entry = $dir_handle->read())) {
               if($entry != '.' && $entry != '..' && substr($entry,0,2) == 'db' && is_file($db_backup_dir.'/'.$entry)) {
                  $files[] = $entry;
               if($entry != '.' && $entry != '..' && preg_match('/^db_(.*?)_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}\.sql.gz$/', $entry, $matches) && is_file($db_backup_dir.'/'.$entry)) {
                        if(array_key_exists($matches[1], $files) == false) $files[$matches[1]] = array();
                  $files[$matches[1]][] = $entry;
               }
            }
            $dir_handle->close();

            rsort($files);

            for ($n = $backup_copies; $n <= 10; $n++) {
               if(isset($files[$n]) && is_file($db_backup_dir.'/'.$files[$n])) {
                  unlink($db_backup_dir.'/'.$files[$n]);
                  $sql = "SELECT backup_id FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($files[$n])."'";
                  $tmp = $app->dbmaster->queryOneRecord($sql);
                  $app->dbmaster->datalogDelete('web_backup', 'backup_id', $tmp['backup_id']);
               }
            }
                
                reset($files);
                foreach($files as $db_name => $filelist) {
                    rsort($filelist);
                    for ($n = $backup_copies; $n <= 10; $n++) {
                        if(isset($filelist[$n]) && is_file($db_backup_dir.'/'.$filelist[$n])) {
                            unlink($db_backup_dir.'/'.$filelist[$n]);
                            //$sql = "SELECT backup_id FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($filelist[$n])."'";
                            //$tmp = $app->dbmaster->queryOneRecord($sql);
                            //$sql = "DELETE FROM web_backup WHERE backup_id = ".intval($tmp['backup_id']);
                     $sql = "DELETE FROM web_backup WHERE server_id = ".$conf['server_id']." AND parent_domain_id = $web_id AND filename = '".$app->db->quote($filelist[$n])."'";
                            $app->db->query($sql);
                            if($app->db->dbHost != $app->dbmaster->dbHost) $app->dbmaster->query($sql);
                        }
                    }
                }

            unset($files);
            unset($dir_handle);