- Fixed "FUNCTION databasename.CONCAT does not exist" error for older MySQL...

ftimme

2013-02-27 f4038a2160d55a7f519a3b42be1aa96e29e9a908

 interface/lib/classes/aps_crawler.inc.php

@@ -34,6 +34,9 @@


class ApsCrawler extends ApsBase

{

   
   //public $app_download_url_list = array();

   
   /**

    * Constructor

    *

@@ -51,6 +54,8 @@
     */

    private function checkRequirements()

    {

        global $app;

        
        try

        {

            // Check if allow_url_fopen is enabled

@@ -58,7 +63,7 @@
            // Check if the cURL module is available

            if(!function_exists('curl_version')) throw new Exception('cURL is not available');

            

            // Check if used folders are writable (chmod 777)

            // Check if used folders are writable

            if($this->interface_mode)

            {

                if(!is_writable($this->interface_pkg_dir)) 

@@ -74,7 +79,7 @@
        }

        catch(Exception $e)

        {

            $this->app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR);

            $app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR);

            return false;

        }

    }

@@ -93,7 +98,7 @@
            foreach($files as $file)

            {

                if($file != '.' && $file != '..')

                    if(filetype($dir.'/'.$file) == 'dir') rrmdir($dir.'/'.$file); 
                    if(filetype($dir.'/'.$file) == 'dir') $this->removeDirectory($dir.'/'.$file); 
                    else @unlink($dir.'/'.$file);

            }

            reset($files);

@@ -197,11 +202,12 @@
     * A method to build query URLs out of a list of vendors

     *

    */

    private function formatVendorCallback(&$array_item, $key)

    private function formatVendorCallback($array_item)

    {

        $array_item = str_replace(' ', '%20', $array_item);

        $array_item = str_replace('http://', '', $array_item);

        $array_item = '/'.$this->aps_version.'.atom?vendor='.$array_item.'&pageSize=100';

      return($array_item);

    }

    

    /**

@@ -209,6 +215,8 @@
     */    

    public function startCrawler() 

    {

        global $app;



        try

        {

            // Make sure the requirements are given so that this script can execute

@@ -216,7 +224,7 @@
            if(!$req_ret) return false;

            

            // Execute the open task and first fetch all vendors (APS catalog API 1.1, p. 12)

            $this->app->log($this->log_prefix.'Fetching data from '.$this->fetch_url);

            $app->log($this->log_prefix.'Fetching data from '.$this->fetch_url);



            $vendor_page = $this->fetchPage('/all-app/'); //$vendor_page = $this->fetchPage('/'.$this->aps_version.'/');

            preg_match_all("/\<a href=\"(.+)\/\" class=\"vendor\"/i", $vendor_page, $matches);

@@ -224,7 +232,12 @@
            if(!$vendors) throw new Exception('Unable to fetch vendors. Aborting');



            // Format all vendors for further processing (i.e. typo3.org -> /1.atom?vendor=typo3.org&pageSize=100

            array_walk($vendors, array($this, 'formatVendorCallback'));

            //array_walk($vendors, array($this, 'formatVendorCallback'));

         if(is_array($vendors)) {

            foreach($vendors as $key => $array_item) {

               $vendors[$key] = $this->formatVendorCallback($array_item);

            }

         }

            

            // Process all vendors in chunks of 50 entries

            $vendor_chunks = array_chunk($vendors, 50);

@@ -233,7 +246,7 @@
            // Get all known apps from the database and the highest known version

            // Note: A dirty hack is used for numerical sorting of the VARCHAR field Version: +0 -> cast

            // A longer but typesafe way would be: ORDER BY CAST(REPLACE(Version, '.', '') AS UNSIGNED) DESC

            $existing_apps = $this->db->queryAllRecords("SELECT * FROM (

            $existing_apps = $app->db->queryAllRecords("SELECT * FROM (

                SELECT name AS Name, CONCAT(version, '-', CAST(`release` AS CHAR)) AS CurrentVersion 

                FROM aps_packages ORDER BY REPLACE(version, '.', '')+0 DESC, `release` DESC

                ) as Versions GROUP BY name");

@@ -270,27 +283,30 @@
                        

                        // Find out a (possibly) existing package version

                        $ex_ver = '';

                  /*

                        array_walk($existing_apps, 

                            create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver);

                        
                        */

                  if(is_array($existing_apps)) {

                     foreach($existing_apps as $k => $v) {

                        if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"];

                     }

                  }

						
                        $new_ver = $app_version.'-'.$app_release;

                        $local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/';



                        // Proceed if a newer or at least equal version has been found with server mode or 

                        // interface mode is activated and there's no valid APP-META.xml existing yet

                        if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0)

                         || ($this->interface_mode 
                              && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0)

                            )

                        )

                        {

                        // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet

                        if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){

                            // Check if we already have an old version of this app

                            if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++; 



                            $app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href");

                            $app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length");

                            $app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href");



							
                     //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl;

                            // Skip ASP.net packages because they can't be used at all

                            $asp_handler = parent::getXPathValue($sxe, '//aspnet:handler');

                            $asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions');

@@ -307,19 +323,22 @@
                                    if(file_exists($old_folder)) $this->removeDirectory($old_folder);

                                    

                           /*

                                    $this->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '".

                                        $this->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".

                                        $this->db->quote($ex_ver)."';");

                                    $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '".

                                        $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".

                                        $app->db->quote($ex_ver)."';");

                           */

                           $tmp = $this->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '".

                                        $this->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".

                                        $this->db->quote($ex_ver)."';");

                           $this->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']);

                           $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '".

                                        $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".

                                        $app->db->quote($ex_ver)."';");

                           $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']);

                           unset($tmp);

                                }

                                

                                // Create the local folder if not yet existing

                                if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true);

								
                        // Save the package URL in an extra file because it's not part of the APP-META.xml file

                        @file_put_contents($local_intf_folder.'PKG_URL', $app_dl);

                                

                                // Download the meta file

                                $local_metafile = $local_intf_folder.'APP-META.xml';

@@ -428,19 +447,19 @@
                    if($apps_to_dl_chunks[$i][$j]['filesize'] != 0 &&

                       $apps_to_dl_chunks[$i][$j]['filesize'] != filesize($apps_to_dl_chunks[$i][$j]['localtarget']))

                    {

                            $this->app->log($this->log_prefix.' The filesize of the package "'.

                            $app->log($this->log_prefix.' The filesize of the package "'.

                                $apps_to_dl_chunks[$i][$j]['name'].'" is wrong. Download failure?', LOGLEVEL_WARN);

                    }

                }

            }

            

            $this->app->log($this->log_prefix.'Processed '.$apps_in_repo.

            $app->log($this->log_prefix.'Processed '.$apps_in_repo.

                ' apps from the repo. Downloaded '.$apps_updated.

                ' updates, '.$apps_downloaded.' new apps');

        }

        catch(Exception $e)

        {

            $this->app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);

            $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);

            return false;

        }

    }

@@ -454,9 +473,11 @@
     */

    public function parseFolderToDB()

    {

        global $app;

        
        try

        {

            // This method must be used in server mode

            // This method must be used in interface mode

            if(!$this->interface_mode) return false; 

            

            $pkg_list = array();

@@ -473,15 +494,16 @@
            

            // Get registered packages and mark non-existant packages with an error code to omit the install

            $existing_packages = array();

            $path_query = $this->db->queryAllRecords('SELECT path AS Path FROM aps_packages;');

            $path_query = $app->db->queryAllRecords('SELECT path AS Path FROM aps_packages;');

            foreach($path_query as $path) $existing_packages[] = $path['Path']; 

            $diff = array_diff($existing_packages, $pkg_list);

            foreach($diff as $todelete)

                /*$this->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."' 
                    WHERE path = '".$this->db->quote($todelete)."';");*/

            $tmp = $this->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$this->db->quote($todelete)."';");

            $this->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']);

            foreach($diff as $todelete) {

                /*$app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."' 
                    WHERE path = '".$app->db->quote($todelete)."';");*/

            $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$app->db->quote($todelete)."';");

            $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']);

            unset($tmp);

         }

            

            // Register all new packages

            $new_packages = array_diff($pkg_list, $existing_packages);

@@ -491,7 +513,7 @@
                $metafile = $this->interface_pkg_dir.'/'.$pkg.'/APP-META.xml';

                if(!file_exists($metafile)) 

                {

                    $this->app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR);

                    $app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR);

                    continue;

                }

        

@@ -506,27 +528,63 @@
                $pkg_category = parent::getXPathValue($sxe, '//category');

                $pkg_version = parent::getXPathValue($sxe, 'version');

                $pkg_release = parent::getXPathValue($sxe, 'release');

            //$pkg_url = $this->app_download_url_list[$pkg];

            $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$pkg.'/PKG_URL');

                

            /*

                $this->db->query("INSERT INTO `aps_packages` 
                $app->db->query("INSERT INTO `aps_packages` 
                    (`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES 

                    ('".$this->db->quote($pkg)."', '".$this->db->quote($pkg_name)."',

                    '".$this->db->quote($pkg_category)."', '".$this->db->quote($pkg_version)."',

                    ".$this->db->quote($pkg_release).", ".PACKAGE_ENABLED.");");

                    ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."',

                    '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."',

                    ".$app->db->quote($pkg_release).", ".PACKAGE_ENABLED.");");

            */

            // Insert only if data is complete

            if($pkg != '' && $pkg_name != '' && $pkg_category != '' && $pkg_version != '' && $pkg_release != '' && $pkg_url){

               $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_url`, `package_status`) VALUES 
                    ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."',

                    '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."',

                    ".$app->db->quote($pkg_release).", '".$app->db->quote($pkg_url)."', ".PACKAGE_ENABLED.");";

            

            $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES 
                    ('".$this->db->quote($pkg)."', '".$this->db->quote($pkg_name)."',

                    '".$this->db->quote($pkg_category)."', '".$this->db->quote($pkg_version)."',

                    ".$this->db->quote($pkg_release).", ".PACKAGE_ENABLED.");";

				
            $app->db->datalogInsert('aps_packages', $insert_data, 'id');

               $app->db->datalogInsert('aps_packages', $insert_data, 'id');

            } else {

               if(file_exists($this->interface_pkg_dir.'/'.$pkg)) $this->removeDirectory($this->interface_pkg_dir.'/'.$pkg);

            }

            }

        }

        catch(Exception $e)

        {

            $this->app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);

         $this->app->error($e->getMessage());

            $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);

         $app->error($e->getMessage());

            return false;

        }

    }

	
   /**

     * Add missing package URLs to database

     */

    public function fixURLs()

    {

        global $app;

        
        try

        {

            // This method must be used in interface mode

            if(!$this->interface_mode) return false; 
            
            $incomplete_pkgs = $app->db->queryAllRecords("SELECT * FROM aps_packages WHERE package_url = ''");

         if(is_array($incomplete_pkgs) && !empty($incomplete_pkgs)){

            foreach($incomplete_pkgs as $incomplete_pkg){

               $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$incomplete_pkg['path'].'/PKG_URL');

               if($pkg_url != ''){

                  $app->db->datalogUpdate('aps_packages', "package_url = '".$pkg_url."'", 'id', $incomplete_pkg['id']);

               }

            }

         }

        }

        catch(Exception $e)

        {

            $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);

         $app->error($e->getMessage());

            return false;

        }

    }