From f4038a2160d55a7f519a3b42be1aa96e29e9a908 Mon Sep 17 00:00:00 2001
From: ftimme <ft@falkotimme.com>
Date: Wed, 27 Feb 2013 09:01:40 -0500
Subject: [PATCH] - Fixed "FUNCTION databasename.CONCAT does not exist" error for older MySQL versions.
---
interface/lib/classes/aps_crawler.inc.php | 150 ++++++++++++++++++++++++++++++++++---------------
1 files changed, 104 insertions(+), 46 deletions(-)
diff --git a/interface/lib/classes/aps_crawler.inc.php b/interface/lib/classes/aps_crawler.inc.php
index 39375f5..00b1d76 100644
--- a/interface/lib/classes/aps_crawler.inc.php
+++ b/interface/lib/classes/aps_crawler.inc.php
@@ -34,6 +34,9 @@
class ApsCrawler extends ApsBase
{
+
+ //public $app_download_url_list = array();
+
/**
* Constructor
*
@@ -51,6 +54,8 @@
*/
private function checkRequirements()
{
+ global $app;
+
try
{
// Check if allow_url_fopen is enabled
@@ -58,7 +63,7 @@
// Check if the cURL module is available
if(!function_exists('curl_version')) throw new Exception('cURL is not available');
- // Check if used folders are writable (chmod 777)
+ // Check if used folders are writable
if($this->interface_mode)
{
if(!is_writable($this->interface_pkg_dir))
@@ -74,7 +79,7 @@
}
catch(Exception $e)
{
- $this->app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR);
+ $app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR);
return false;
}
}
@@ -93,7 +98,7 @@
foreach($files as $file)
{
if($file != '.' && $file != '..')
- if(filetype($dir.'/'.$file) == 'dir') rrmdir($dir.'/'.$file);
+ if(filetype($dir.'/'.$file) == 'dir') $this->removeDirectory($dir.'/'.$file);
else @unlink($dir.'/'.$file);
}
reset($files);
@@ -197,11 +202,12 @@
* A method to build query URLs out of a list of vendors
*
*/
- private function formatVendorCallback(&$array_item, $key)
+ private function formatVendorCallback($array_item)
{
$array_item = str_replace(' ', '%20', $array_item);
$array_item = str_replace('http://', '', $array_item);
$array_item = '/'.$this->aps_version.'.atom?vendor='.$array_item.'&pageSize=100';
+ return($array_item);
}
/**
@@ -209,6 +215,8 @@
*/
public function startCrawler()
{
+ global $app;
+
try
{
// Make sure the requirements are given so that this script can execute
@@ -216,7 +224,7 @@
if(!$req_ret) return false;
// Execute the open task and first fetch all vendors (APS catalog API 1.1, p. 12)
- $this->app->log($this->log_prefix.'Fetching data from '.$this->fetch_url);
+ $app->log($this->log_prefix.'Fetching data from '.$this->fetch_url);
$vendor_page = $this->fetchPage('/all-app/'); //$vendor_page = $this->fetchPage('/'.$this->aps_version.'/');
preg_match_all("/\<a href=\"(.+)\/\" class=\"vendor\"/i", $vendor_page, $matches);
@@ -224,7 +232,12 @@
if(!$vendors) throw new Exception('Unable to fetch vendors. Aborting');
// Format all vendors for further processing (i.e. typo3.org -> /1.atom?vendor=typo3.org&pageSize=100
- array_walk($vendors, array($this, 'formatVendorCallback'));
+ //array_walk($vendors, array($this, 'formatVendorCallback'));
+ if(is_array($vendors)) {
+ foreach($vendors as $key => $array_item) {
+ $vendors[$key] = $this->formatVendorCallback($array_item);
+ }
+ }
// Process all vendors in chunks of 50 entries
$vendor_chunks = array_chunk($vendors, 50);
@@ -233,7 +246,7 @@
// Get all known apps from the database and the highest known version
// Note: A dirty hack is used for numerical sorting of the VARCHAR field Version: +0 -> cast
// A longer but typesafe way would be: ORDER BY CAST(REPLACE(Version, '.', '') AS UNSIGNED) DESC
- $existing_apps = $this->db->queryAllRecords("SELECT * FROM (
+ $existing_apps = $app->db->queryAllRecords("SELECT * FROM (
SELECT name AS Name, CONCAT(version, '-', CAST(`release` AS CHAR)) AS CurrentVersion
FROM aps_packages ORDER BY REPLACE(version, '.', '')+0 DESC, `release` DESC
) as Versions GROUP BY name");
@@ -270,27 +283,30 @@
// Find out a (possibly) existing package version
$ex_ver = '';
+ /*
array_walk($existing_apps,
create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver);
-
+ */
+ if(is_array($existing_apps)) {
+ foreach($existing_apps as $k => $v) {
+ if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"];
+ }
+ }
+
$new_ver = $app_version.'-'.$app_release;
$local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/';
// Proceed if a newer or at least equal version has been found with server mode or
- // interface mode is activated and there's no valid APP-META.xml existing yet
- if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0)
- || ($this->interface_mode
- && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0)
- )
- )
- {
+ // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet
+ if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){
// Check if we already have an old version of this app
if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++;
$app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href");
$app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length");
$app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href");
-
+
+ //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl;
// Skip ASP.net packages because they can't be used at all
$asp_handler = parent::getXPathValue($sxe, '//aspnet:handler');
$asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions');
@@ -307,19 +323,22 @@
if(file_exists($old_folder)) $this->removeDirectory($old_folder);
/*
- $this->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '".
- $this->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".
- $this->db->quote($ex_ver)."';");
+ $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '".
+ $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".
+ $app->db->quote($ex_ver)."';");
*/
- $tmp = $this->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '".
- $this->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".
- $this->db->quote($ex_ver)."';");
- $this->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']);
+ $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '".
+ $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '".
+ $app->db->quote($ex_ver)."';");
+ $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']);
unset($tmp);
}
// Create the local folder if not yet existing
if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true);
+
+ // Save the package URL in an extra file because it's not part of the APP-META.xml file
+ @file_put_contents($local_intf_folder.'PKG_URL', $app_dl);
// Download the meta file
$local_metafile = $local_intf_folder.'APP-META.xml';
@@ -428,19 +447,19 @@
if($apps_to_dl_chunks[$i][$j]['filesize'] != 0 &&
$apps_to_dl_chunks[$i][$j]['filesize'] != filesize($apps_to_dl_chunks[$i][$j]['localtarget']))
{
- $this->app->log($this->log_prefix.' The filesize of the package "'.
+ $app->log($this->log_prefix.' The filesize of the package "'.
$apps_to_dl_chunks[$i][$j]['name'].'" is wrong. Download failure?', LOGLEVEL_WARN);
}
}
}
- $this->app->log($this->log_prefix.'Processed '.$apps_in_repo.
+ $app->log($this->log_prefix.'Processed '.$apps_in_repo.
' apps from the repo. Downloaded '.$apps_updated.
' updates, '.$apps_downloaded.' new apps');
}
catch(Exception $e)
{
- $this->app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);
+ $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);
return false;
}
}
@@ -454,9 +473,11 @@
*/
public function parseFolderToDB()
{
+ global $app;
+
try
{
- // This method must be used in server mode
+ // This method must be used in interface mode
if(!$this->interface_mode) return false;
$pkg_list = array();
@@ -473,15 +494,16 @@
// Get registered packages and mark non-existant packages with an error code to omit the install
$existing_packages = array();
- $path_query = $this->db->queryAllRecords('SELECT path AS Path FROM aps_packages;');
+ $path_query = $app->db->queryAllRecords('SELECT path AS Path FROM aps_packages;');
foreach($path_query as $path) $existing_packages[] = $path['Path'];
$diff = array_diff($existing_packages, $pkg_list);
- foreach($diff as $todelete)
- /*$this->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."'
- WHERE path = '".$this->db->quote($todelete)."';");*/
- $tmp = $this->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$this->db->quote($todelete)."';");
- $this->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']);
+ foreach($diff as $todelete) {
+ /*$app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."'
+ WHERE path = '".$app->db->quote($todelete)."';");*/
+ $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$app->db->quote($todelete)."';");
+ $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']);
unset($tmp);
+ }
// Register all new packages
$new_packages = array_diff($pkg_list, $existing_packages);
@@ -491,7 +513,7 @@
$metafile = $this->interface_pkg_dir.'/'.$pkg.'/APP-META.xml';
if(!file_exists($metafile))
{
- $this->app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR);
+ $app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR);
continue;
}
@@ -506,27 +528,63 @@
$pkg_category = parent::getXPathValue($sxe, '//category');
$pkg_version = parent::getXPathValue($sxe, 'version');
$pkg_release = parent::getXPathValue($sxe, 'release');
+ //$pkg_url = $this->app_download_url_list[$pkg];
+ $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$pkg.'/PKG_URL');
/*
- $this->db->query("INSERT INTO `aps_packages`
+ $app->db->query("INSERT INTO `aps_packages`
(`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES
- ('".$this->db->quote($pkg)."', '".$this->db->quote($pkg_name)."',
- '".$this->db->quote($pkg_category)."', '".$this->db->quote($pkg_version)."',
- ".$this->db->quote($pkg_release).", ".PACKAGE_ENABLED.");");
+ ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."',
+ '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."',
+ ".$app->db->quote($pkg_release).", ".PACKAGE_ENABLED.");");
*/
+ // Insert only if data is complete
+ if($pkg != '' && $pkg_name != '' && $pkg_category != '' && $pkg_version != '' && $pkg_release != '' && $pkg_url){
+ $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_url`, `package_status`) VALUES
+ ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."',
+ '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."',
+ ".$app->db->quote($pkg_release).", '".$app->db->quote($pkg_url)."', ".PACKAGE_ENABLED.");";
- $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES
- ('".$this->db->quote($pkg)."', '".$this->db->quote($pkg_name)."',
- '".$this->db->quote($pkg_category)."', '".$this->db->quote($pkg_version)."',
- ".$this->db->quote($pkg_release).", ".PACKAGE_ENABLED.");";
-
- $this->app->db->datalogInsert('aps_packages', $insert_data, 'id');
+ $app->db->datalogInsert('aps_packages', $insert_data, 'id');
+ } else {
+ if(file_exists($this->interface_pkg_dir.'/'.$pkg)) $this->removeDirectory($this->interface_pkg_dir.'/'.$pkg);
+ }
}
}
catch(Exception $e)
{
- $this->app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);
- $this->app->error($e->getMessage());
+ $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);
+ $app->error($e->getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Add missing package URLs to database
+ */
+ public function fixURLs()
+ {
+ global $app;
+
+ try
+ {
+ // This method must be used in interface mode
+ if(!$this->interface_mode) return false;
+
+ $incomplete_pkgs = $app->db->queryAllRecords("SELECT * FROM aps_packages WHERE package_url = ''");
+ if(is_array($incomplete_pkgs) && !empty($incomplete_pkgs)){
+ foreach($incomplete_pkgs as $incomplete_pkg){
+ $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$incomplete_pkg['path'].'/PKG_URL');
+ if($pkg_url != ''){
+ $app->db->datalogUpdate('aps_packages', "package_url = '".$pkg_url."'", 'id', $incomplete_pkg['id']);
+ }
+ }
+ }
+ }
+ catch(Exception $e)
+ {
+ $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR);
+ $app->error($e->getMessage());
return false;
}
}
--
Gitblit v1.9.1