From kde-commits Sat Mar 31 20:53:22 2012 From: Ingo Malchow Date: Sat, 31 Mar 2012 20:53:22 +0000 To: kde-commits Subject: [websites/wiki-kde-org/develop] includes/cache: force to add includes cache Message-Id: <20120331205322.02505A60A9 () git ! kde ! org> X-MARC-Message: https://marc.info/?l=kde-commits&m=133322729613299 Git commit 6c998f2fd798b527bd184cbb7286d874f89abea8 by Ingo Malchow. Committed on 31/03/2012 at 22:53. Pushed by imalchow into branch 'develop'. force to add includes cache A +410 -0 includes/cache/CacheDependency.php A +249 -0 includes/cache/FileCacheBase.php A +135 -0 includes/cache/GenderCache.php A +239 -0 includes/cache/HTMLCacheUpdate.php A +188 -0 includes/cache/HTMLFileCache.php A +207 -0 includes/cache/LinkBatch.php A +216 -0 includes/cache/LinkCache.php A +98 -0 includes/cache/MemcachedSessions.php A +999 -0 includes/cache/MessageCache.php A +30 -0 includes/cache/ObjectFileCache.php A +87 -0 includes/cache/ResourceFileCache.php A +226 -0 includes/cache/SquidUpdate.php http://commits.kde.org/websites/wiki-kde-org/6c998f2fd798b527bd184cbb7286d8= 74f89abea8 diff --git a/includes/cache/CacheDependency.php b/includes/cache/CacheDepen= dency.php new file mode 100644 index 0000000..0df0cd8 --- /dev/null +++ b/includes/cache/CacheDependency.php @@ -0,0 +1,410 @@ +value =3D $value; + + if ( !is_array( $deps ) ) { + $deps =3D array( $deps ); + } + + $this->deps =3D $deps; + } + + /** + * Returns true if any of the dependencies have expired + * + * @return bool + */ + function isExpired() { + foreach ( $this->deps as $dep ) { + if ( $dep->isExpired() ) { + return true; + } + } + + return false; + } + + /** + * Initialise dependency values in preparation for storing. This must be + * called before serialization. + */ + function initialiseDeps() { + foreach ( $this->deps as $dep ) { + $dep->loadDependencyValues(); + } + } + + /** + * Get the user-defined value + * @return bool|\Mixed + */ + function getValue() { + return $this->value; + } + + /** + * Store the wrapper to a cache + * + * @param $cache BagOStuff + * @param $key + * @param $expiry + */ + function storeToCache( $cache, $key, $expiry =3D 0 ) { + $this->initialiseDeps(); + $cache->set( $key, $this, $expiry ); + } + + /** + * Attempt to get a value from the cache. If the value is expired or miss= ing, + * it will be generated with the callback function (if present), and the = newly + * calculated value will be stored to the cache in a wrapper. + * + * @param $cache BagOStuff a cache object such as $wgMemc + * @param $key String: the cache key + * @param $expiry Integer: the expiry timestamp or interval in seconds + * @param $callback Mixed: the callback for generating the value, or false + * @param $callbackParams Array: the function parameters for the callback + * @param $deps Array: the dependencies to store on a cache miss. Note: t= hese + * are not the dependencies used on a cache hit! Cache hits use the st= ored + * dependency array. + * + * @return mixed The value, or null if it was not present in the cache an= d no + * callback was defined. + */ + static function getValueFromCache( $cache, $key, $expiry =3D 0, $callback= =3D false, + $callbackParams =3D array(), $deps =3D array() ) + { + $obj =3D $cache->get( $key ); + + if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->is= Expired() ) { + $value =3D $obj->value; + } elseif ( $callback ) { + $value =3D call_user_func_array( $callback, $callbackParams ); + # Cache the newly-generated value + $wrapper =3D new DependencyWrapper( $value, $deps ); + $wrapper->storeToCache( $cache, $key, $expiry ); + } else { + $value =3D null; + } + + return $value; + } +} + +/** + * @ingroup Cache + */ +abstract class CacheDependency { + /** + * Returns true if the dependency is expired, false otherwise + */ + abstract function isExpired(); + + /** + * Hook to perform any expensive pre-serialize loading of dependency valu= es. + */ + function loadDependencyValues() { } +} + +/** + * @ingroup Cache + */ +class FileDependency extends CacheDependency { + var $filename, $timestamp; + + /** + * Create a file dependency + * + * @param $filename String: the name of the file, preferably fully qualif= ied + * @param $timestamp Mixed: the unix last modified timestamp, or false if= the + * file does not exist. If omitted, the timestamp will be loaded f= rom + * the file. + * + * A dependency on a nonexistent file will be triggered when the file is + * created. A dependency on an existing file will be triggered when the + * file is changed. + */ + function __construct( $filename, $timestamp =3D null ) { + $this->filename =3D $filename; + $this->timestamp =3D $timestamp; + } + + /** + * @return array + */ + function __sleep() { + $this->loadDependencyValues(); + return array( 'filename', 'timestamp' ); + } + + function loadDependencyValues() { + if ( is_null( $this->timestamp ) ) { + if ( !file_exists( $this->filename ) ) { + # Dependency on a non-existent file + # This is a valid concept! + $this->timestamp =3D false; + } else { + $this->timestamp =3D filemtime( $this->filename ); + } + } + } + + /** + * @return bool + */ + function isExpired() { + if ( !file_exists( $this->filename ) ) { + if ( $this->timestamp =3D=3D=3D false ) { + # Still nonexistent + return false; + } else { + # Deleted + wfDebug( "Dependency triggered: {$this->filename} deleted.\n" ); + return true; + } + } else { + $lastmod =3D filemtime( $this->filename ); + if ( $lastmod > $this->timestamp ) { + # Modified or created + wfDebug( "Dependency triggered: {$this->filename} changed.\n" ); + return true; + } else { + # Not modified + return false; + } + } + } +} + +/** + * @ingroup Cache + */ +class TitleDependency extends CacheDependency { + var $titleObj; + var $ns, $dbk; + var $touched; + + /** + * Construct a title dependency + * @param $title Title + */ + function __construct( Title $title ) { + $this->titleObj =3D $title; + $this->ns =3D $title->getNamespace(); + $this->dbk =3D $title->getDBkey(); + } + + function loadDependencyValues() { + $this->touched =3D $this->getTitle()->getTouched(); + } + + /** + * Get rid of bulky Title object for sleep + * + * @return array + */ + function __sleep() { + return array( 'ns', 'dbk', 'touched' ); + } + + /** + * @return Title + */ + function getTitle() { + if ( !isset( $this->titleObj ) ) { + $this->titleObj =3D Title::makeTitle( $this->ns, $this->dbk ); + } + + return $this->titleObj; + } + + /** + * @return bool + */ + function isExpired() { + $touched =3D $this->getTitle()->getTouched(); + + if ( $this->touched =3D=3D=3D false ) { + if ( $touched =3D=3D=3D false ) { + # Still missing + return false; + } else { + # Created + return true; + } + } elseif ( $touched =3D=3D=3D false ) { + # Deleted + return true; + } elseif ( $touched > $this->touched ) { + # Updated + return true; + } else { + # Unmodified + return false; + } + } +} + +/** + * @ingroup Cache + */ +class TitleListDependency extends CacheDependency { + var $linkBatch; + var $timestamps; + + /** + * Construct a dependency on a list of titles + * @param $linkBatch LinkBatch + */ + function __construct( LinkBatch $linkBatch ) { + $this->linkBatch =3D $linkBatch; + } + + /** + * @return array + */ + function calculateTimestamps() { + # Initialise values to false + $timestamps =3D array(); + + foreach ( $this->getLinkBatch()->data as $ns =3D> $dbks ) { + if ( count( $dbks ) > 0 ) { + $timestamps[$ns] =3D array(); + + foreach ( $dbks as $dbk =3D> $value ) { + $timestamps[$ns][$dbk] =3D false; + } + } + } + + # Do the query + if ( count( $timestamps ) ) { + $dbr =3D wfGetDB( DB_SLAVE ); + $where =3D $this->getLinkBatch()->constructSet( 'page', $dbr ); + $res =3D $dbr->select( + 'page', + array( 'page_namespace', 'page_title', 'page_touched' ), + $where, + __METHOD__ + ); + + foreach ( $res as $row ) { + $timestamps[$row->page_namespace][$row->page_title] =3D $row->page_tou= ched; + } + } + + return $timestamps; + } + + function loadDependencyValues() { + $this->timestamps =3D $this->calculateTimestamps(); + } + + /** + * @return array + */ + function __sleep() { + return array( 'timestamps' ); + } + + /** + * @return LinkBatch + */ + function getLinkBatch() { + if ( !isset( $this->linkBatch ) ) { + $this->linkBatch =3D new LinkBatch; + $this->linkBatch->setArray( $this->timestamps ); + } + return $this->linkBatch; + } + + /** + * @return bool + */ + function isExpired() { + $newTimestamps =3D $this->calculateTimestamps(); + + foreach ( $this->timestamps as $ns =3D> $dbks ) { + foreach ( $dbks as $dbk =3D> $oldTimestamp ) { + $newTimestamp =3D $newTimestamps[$ns][$dbk]; + + if ( $oldTimestamp =3D=3D=3D false ) { + if ( $newTimestamp =3D=3D=3D false ) { + # Still missing + } else { + # Created + return true; + } + } elseif ( $newTimestamp =3D=3D=3D false ) { + # Deleted + return true; + } elseif ( $newTimestamp > $oldTimestamp ) { + # Updated + return true; + } else { + # Unmodified + } + } + } + + return false; + } +} + +/** + * @ingroup Cache + */ +class GlobalDependency extends CacheDependency { + var $name, $value; + + function __construct( $name ) { + $this->name =3D $name; + $this->value =3D $GLOBALS[$name]; + } + + /** + * @return bool + */ + function isExpired() { + if( !isset($GLOBALS[$this->name]) ) { + return true; + } + return $GLOBALS[$this->name] !=3D $this->value; + } +} + +/** + * @ingroup Cache + */ +class ConstantDependency extends CacheDependency { + var $name, $value; + + function __construct( $name ) { + $this->name =3D $name; + $this->value =3D constant( $name ); + } + + /** + * @return bool + */ + function isExpired() { + return constant( $this->name ) !=3D $this->value; + } +} diff --git a/includes/cache/FileCacheBase.php b/includes/cache/FileCacheBas= e.php new file mode 100644 index 0000000..3740165 --- /dev/null +++ b/includes/cache/FileCacheBase.php @@ -0,0 +1,249 @@ +mUseGzip =3D (bool)$wgUseGzip; + } + + /** + * Get the base file cache directory + * @return string + */ + final protected function baseCacheDirectory() { + global $wgFileCacheDirectory; + return $wgFileCacheDirectory; + } + + /** + * Get the base cache directory (not specific to this file) + * @return string + */ + abstract protected function cacheDirectory(); + + /** + * Get the path to the cache file + * @return string + */ + protected function cachePath() { + if ( $this->mFilePath !=3D=3D null ) { + return $this->mFilePath; + } + + $dir =3D $this->cacheDirectory(); + # Build directories (methods include the trailing "/") + $subDirs =3D $this->typeSubdirectory() . $this->hashSubdirectory(); + # Avoid extension confusion + $key =3D str_replace( '.', '%2E', urlencode( $this->mKey ) ); + # Build the full file path + $this->mFilePath =3D "{$dir}/{$subDirs}{$key}.{$this->mExt}"; + if ( $this->useGzip() ) { + $this->mFilePath .=3D '.gz'; + } + + return $this->mFilePath; + } + + /** + * Check if the cache file exists + * @return bool + */ + public function isCached() { + if ( $this->mCached =3D=3D=3D null ) { + $this->mCached =3D file_exists( $this->cachePath() ); + } + return $this->mCached; + } + + /** + * Get the last-modified timestamp of the cache file + * @return string|false TS_MW timestamp + */ + public function cacheTimestamp() { + $timestamp =3D filemtime( $this->cachePath() ); + return ( $timestamp !=3D=3D false ) + ? wfTimestamp( TS_MW, $timestamp ) + : false; + } + + /** + * Check if up to date cache file exists + * @param $timestamp string MW_TS timestamp + * + * @return bool + */ + public function isCacheGood( $timestamp =3D '' ) { + global $wgCacheEpoch; + + if ( !$this->isCached() ) { + return false; + } + + $cachetime =3D $this->cacheTimestamp(); + $good =3D ( $timestamp <=3D $cachetime && $wgCacheEpoch <=3D $cachetime = ); + wfDebug( __METHOD__ . ": cachetime $cachetime, touched '{$timestamp}' ep= och {$wgCacheEpoch}, good $good\n" ); + + return $good; + } + + /** + * Check if the cache is gzipped + * @return bool + */ + protected function useGzip() { + return $this->mUseGzip; + } + + /** + * Get the uncompressed text from the cache + * @return string + */ + public function fetchText() { + // gzopen can transparently read from gziped or plain text + $fh =3D gzopen( $this->cachePath(), 'rb' ); + return stream_get_contents( $fh ); + } + + /** + * Save and compress text to the cache + * @return string compressed text + */ + public function saveText( $text ) { + global $wgUseFileCache; + + if ( !$wgUseFileCache ) { + return false; + } + + if ( $this->useGzip() ) { + $text =3D gzencode( $text ); + } + + $this->checkCacheDirs(); // build parent dir + if ( !file_put_contents( $this->cachePath(), $text, LOCK_EX ) ) { + wfDebug( __METHOD__ . "() failed saving ". $this->cachePath() . "\n"); + $this->mCached =3D null; + return false; + } + + $this->mCached =3D true; + return $text; + } + + /** + * Clear the cache for this page + * @return void + */ + public function clearCache() { + wfSuppressWarnings(); + unlink( $this->cachePath() ); + wfRestoreWarnings(); + $this->mCached =3D false; + } + + /** + * Create parent directors of $this->cachePath() + * @return void + */ + protected function checkCacheDirs() { + wfMkdirParents( dirname( $this->cachePath() ), null, __METHOD__ ); + } + + /** + * Get the cache type subdirectory (with trailing slash) + * An extending class could use that method to alter the type -> directory + * mapping. @see HTMLFileCache::typeSubdirectory() for an example. + * + * @return string + */ + protected function typeSubdirectory() { + return $this->mType . '/'; + } + + /** + * Return relative multi-level hash subdirectory (with trailing slash) + * or the empty string if not $wgFileCacheDepth + * @return string + */ + protected function hashSubdirectory() { + global $wgFileCacheDepth; + + $subdir =3D ''; + if ( $wgFileCacheDepth > 0 ) { + $hash =3D md5( $this->mKey ); + for ( $i =3D 1; $i <=3D $wgFileCacheDepth; $i++ ) { + $subdir .=3D substr( $hash, 0, $i ) . '/'; + } + } + + return $subdir; + } + + /** + * Roughly increments the cache misses in the last hour by unique visitors + * @param $request WebRequest + * @return void + */ + public function incrMissesRecent( WebRequest $request ) { + global $wgMemc; + if ( mt_rand( 0, self::MISS_FACTOR - 1 ) =3D=3D 0 ) { + # Get a large IP range that should include the user even if that = + # person's IP address changes + $ip =3D $request->getIP(); + if ( !IP::isValid( $ip ) ) { + return; + } + $ip =3D IP::isIPv6( $ip ) + ? IP::sanitizeRange( "$ip/32" ) + : IP::sanitizeRange( "$ip/16" ); + + # Bail out if a request already came from this range... + $key =3D wfMemcKey( get_class( $this ), 'attempt', $this->mType, $this-= >mKey, $ip ); + if ( $wgMemc->get( $key ) ) { + return; // possibly the same user + } + $wgMemc->set( $key, 1, self::MISS_TTL_SEC ); + + # Increment the number of cache misses... + $key =3D $this->cacheMissKey(); + if ( $wgMemc->get( $key ) =3D=3D=3D false ) { + $wgMemc->set( $key, 1, self::MISS_TTL_SEC ); + } else { + $wgMemc->incr( $key ); + } + } + } + + /** + * Roughly gets the cache misses in the last hour by unique visitors + * @return int + */ + public function getMissesRecent() { + global $wgMemc; + return self::MISS_FACTOR * $wgMemc->get( $this->cacheMissKey() ); + } + + /** + * @return string + */ + protected function cacheMissKey() { + return wfMemcKey( get_class( $this ), 'misses', $this->mType, $this->mKe= y ); + } +} diff --git a/includes/cache/GenderCache.php b/includes/cache/GenderCache.php new file mode 100644 index 0000000..342f8db --- /dev/null +++ b/includes/cache/GenderCache.php @@ -0,0 +1,135 @@ +default =3D=3D=3D null ) { + $this->default =3D User::getDefaultOption( 'gender' ); + } + return $this->default; + } + + /** + * Returns the gender for given username. + * @param $username String: username + * @param $caller String: the calling method + * @return String + */ + public function getGenderOf( $username, $caller =3D '' ) { + global $wgUser; + + $username =3D strtr( $username, '_', ' ' ); + if ( !isset( $this->cache[$username] ) ) { + + if ( $this->misses >=3D $this->missLimit && $wgUser->getName() !=3D=3D = $username ) { + if( $this->misses =3D=3D=3D $this->missLimit ) { + $this->misses++; + wfDebug( __METHOD__ . ": too many misses, returning default onwards\n= " ); + } + return $this->getDefault(); + + } else { + $this->misses++; + if ( !User::isValidUserName( $username ) ) { + $this->cache[$username] =3D $this->getDefault(); + } else { + $this->doQuery( $username, $caller ); + } + } + + } + + /* Undefined if there is a valid username which for some reason doesn't + * exist in the database. + */ + return isset( $this->cache[$username] ) ? $this->cache[$username] : $thi= s->getDefault(); + } + + /** + * Wrapper for doQuery that processes raw LinkBatch data. + * + * @param $data + * @param $caller + */ + public function doLinkBatch( $data, $caller =3D '' ) { + $users =3D array(); + foreach ( $data as $ns =3D> $pagenames ) { + if ( !MWNamespace::hasGenderDistinction( $ns ) ) continue; + foreach ( array_keys( $pagenames ) as $username ) { + if ( isset( $this->cache[$username] ) ) continue; + $users[$username] =3D true; + } + } + + $this->doQuery( array_keys( $users ), $caller ); + } + + /** + * Preloads genders for given list of users. + * @param $users List|String: usernames + * @param $caller String: the calling method + */ + public function doQuery( $users, $caller =3D '' ) { + $default =3D $this->getDefault(); + + foreach ( (array) $users as $index =3D> $value ) { + $name =3D strtr( $value, '_', ' ' ); + if ( isset( $this->cache[$name] ) ) { + // Skip users whose gender setting we already know + unset( $users[$index] ); + } else { + $users[$index] =3D $name; + // For existing users, this value will be overwritten by the correct v= alue + $this->cache[$name] =3D $default; + } + } + + if ( count( $users ) =3D=3D=3D 0 ) { + return; + } + + $dbr =3D wfGetDB( DB_SLAVE ); + $table =3D array( 'user', 'user_properties' ); + $fields =3D array( 'user_name', 'up_value' ); + $conds =3D array( 'user_name' =3D> $users ); + $joins =3D array( 'user_properties' =3D> + array( 'LEFT JOIN', array( 'user_id =3D up_user', 'up_property' =3D> 'g= ender' ) ) ); + + $comment =3D __METHOD__; + if ( strval( $caller ) !=3D=3D '' ) { + $comment .=3D "/$caller"; + } + $res =3D $dbr->select( $table, $fields, $conds, $comment, $joins, $joins= ); + + foreach ( $res as $row ) { + $this->cache[$row->user_name] =3D $row->up_value ? $row->up_value : $de= fault; + } + } + +} diff --git a/includes/cache/HTMLCacheUpdate.php b/includes/cache/HTMLCacheU= pdate.php new file mode 100644 index 0000000..11e2ae7 --- /dev/null +++ b/includes/cache/HTMLCacheUpdate.php @@ -0,0 +1,239 @@ +mTitle =3D $titleTo; + $this->mTable =3D $table; + $this->mStart =3D $start; + $this->mEnd =3D $end; + $this->mRowsPerJob =3D $wgUpdateRowsPerJob; + $this->mRowsPerQuery =3D $wgUpdateRowsPerQuery; + $this->mCache =3D $this->mTitle->getBacklinkCache(); + } + + public function doUpdate() { + if ( $this->mStart || $this->mEnd ) { + $this->doPartialUpdate(); + return; + } + + # Get an estimate of the number of rows from the BacklinkCache + $numRows =3D $this->mCache->getNumLinks( $this->mTable ); + if ( $numRows > $this->mRowsPerJob * 2 ) { + # Do fast cached partition + $this->insertJobs(); + } else { + # Get the links from the DB + $titleArray =3D $this->mCache->getLinks( $this->mTable ); + # Check if the row count estimate was correct + if ( $titleArray->count() > $this->mRowsPerJob * 2 ) { + # Not correct, do accurate partition + wfDebug( __METHOD__.": row count estimate was incorrect, repartitionin= g\n" ); + $this->insertJobsFromTitles( $titleArray ); + } else { + $this->invalidateTitles( $titleArray ); + } + } + } + + /** + * Update some of the backlinks, defined by a page ID range + */ + protected function doPartialUpdate() { + $titleArray =3D $this->mCache->getLinks( $this->mTable, $this->mStart, $= this->mEnd ); + if ( $titleArray->count() <=3D $this->mRowsPerJob * 2 ) { + # This partition is small enough, do the update + $this->invalidateTitles( $titleArray ); + } else { + # Partitioning was excessively inaccurate. Divide the job further. + # This can occur when a large number of links are added in a short + # period of time, say by updating a heavily-used template. + $this->insertJobsFromTitles( $titleArray ); + } + } + + /** + * Partition the current range given by $this->mStart and $this->mEnd, + * using a pre-calculated title array which gives the links in that range. + * Queue the resulting jobs. + * + * @param $titleArray array + */ + protected function insertJobsFromTitles( $titleArray ) { + # We make subpartitions in the sense that the start of the first job + # will be the start of the parent partition, and the end of the last + # job will be the end of the parent partition. + $jobs =3D array(); + $start =3D $this->mStart; # start of the current job + $numTitles =3D 0; + foreach ( $titleArray as $title ) { + $id =3D $title->getArticleID(); + # $numTitles is now the number of titles in the current job not + # including the current ID + if ( $numTitles >=3D $this->mRowsPerJob ) { + # Add a job up to but not including the current ID + $params =3D array( + 'table' =3D> $this->mTable, + 'start' =3D> $start, + 'end' =3D> $id - 1 + ); + $jobs[] =3D new HTMLCacheUpdateJob( $this->mTitle, $params ); + $start =3D $id; + $numTitles =3D 0; + } + $numTitles++; + } + # Last job + $params =3D array( + 'table' =3D> $this->mTable, + 'start' =3D> $start, + 'end' =3D> $this->mEnd + ); + $jobs[] =3D new HTMLCacheUpdateJob( $this->mTitle, $params ); + wfDebug( __METHOD__.": repartitioning into " . count( $jobs ) . " jobs\n= " ); + + if ( count( $jobs ) < 2 ) { + # I don't think this is possible at present, but handling this case + # makes the code a bit more robust against future code updates and + # avoids a potential infinite loop of repartitioning + wfDebug( __METHOD__.": repartitioning failed!\n" ); + $this->invalidateTitles( $titleArray ); + return; + } + + Job::batchInsert( $jobs ); + } + + /** + * @return mixed + */ + protected function insertJobs() { + $batches =3D $this->mCache->partition( $this->mTable, $this->mRowsPerJob= ); + if ( !$batches ) { + return; + } + $jobs =3D array(); + foreach ( $batches as $batch ) { + $params =3D array( + 'table' =3D> $this->mTable, + 'start' =3D> $batch[0], + 'end' =3D> $batch[1], + ); + $jobs[] =3D new HTMLCacheUpdateJob( $this->mTitle, $params ); + } + Job::batchInsert( $jobs ); + } + + /** + * Invalidate an array (or iterator) of Title objects, right now + * @param $titleArray array + */ + protected function invalidateTitles( $titleArray ) { + global $wgUseFileCache, $wgUseSquid; + + $dbw =3D wfGetDB( DB_MASTER ); + $timestamp =3D $dbw->timestamp(); + + # Get all IDs in this query into an array + $ids =3D array(); + foreach ( $titleArray as $title ) { + $ids[] =3D $title->getArticleID(); + } + + if ( !$ids ) { + return; + } + + # Update page_touched + $batches =3D array_chunk( $ids, $this->mRowsPerQuery ); + foreach ( $batches as $batch ) { + $dbw->update( 'page', + array( 'page_touched' =3D> $timestamp ), + array( 'page_id' =3D> $batch ), + __METHOD__ + ); + } + + # Update squid + if ( $wgUseSquid ) { + $u =3D SquidUpdate::newFromTitles( $titleArray ); + $u->doUpdate(); + } + + # Update file cache + if ( $wgUseFileCache ) { + foreach ( $titleArray as $title ) { + HTMLFileCache::clearFileCache( $title ); + } + } + } +} + + +/** + * Job wrapper for HTMLCacheUpdate. Gets run whenever a related + * job gets called from the queue. + * + * @ingroup JobQueue + */ +class HTMLCacheUpdateJob extends Job { + var $table, $start, $end; + + /** + * Construct a job + * @param $title Title: the title linked to + * @param $params Array: job parameters (table, start and end page_ids) + * @param $id Integer: job id + */ + function __construct( $title, $params, $id =3D 0 ) { + parent::__construct( 'htmlCacheUpdate', $title, $params, $id ); + $this->table =3D $params['table']; + $this->start =3D $params['start']; + $this->end =3D $params['end']; + } + + public function run() { + $update =3D new HTMLCacheUpdate( $this->title, $this->table, $this->star= t, $this->end ); + $update->doUpdate(); + return true; + } +} diff --git a/includes/cache/HTMLFileCache.php b/includes/cache/HTMLFileCach= e.php new file mode 100644 index 0000000..92130f6 --- /dev/null +++ b/includes/cache/HTMLFileCache.php @@ -0,0 +1,188 @@ +mKey =3D ( $title instanceof Title ) + ? $title->getPrefixedDBkey() + : (string)$title; + $cache->mType =3D (string)$action; + $cache->mExt =3D 'html'; + + return $cache; + } + + /** + * Cacheable actions + * @return array + */ + protected static function cacheablePageActions() { + return array( 'view', 'history' ); + } + + /** + * Get the base file cache directory + * @return string + */ + protected function cacheDirectory() { + return $this->baseCacheDirectory(); // no subdir for b/c with old cache = files + } + + /** + * Get the cache type subdirectory (with the trailing slash) or the empty= string + * Alter the type -> directory mapping to put action=3Dview cache at the = root. + * + * @return string + */ + protected function typeSubdirectory() { + if ( $this->mType =3D=3D=3D 'view' ) { + return ''; // b/c to not skip existing cache + } else { + return $this->mType . '/'; + } + } + + /** + * Check if pages can be cached for this request/user + * @param $context IContextSource + * @return bool + */ + public static function useFileCache( IContextSource $context ) { + global $wgUseFileCache, $wgShowIPinHeader, $wgDebugToolbar, $wgContLang; + if ( !$wgUseFileCache ) { + return false; + } + if ( $wgShowIPinHeader || $wgDebugToolbar ) { + wfDebug( "HTML file cache skipped. Either \$wgShowIPinHeader and/or \$w= gDebugToolbar on\n" ); + return false; + } + + // Get all query values + $queryVals =3D $context->getRequest()->getValues(); + foreach ( $queryVals as $query =3D> $val ) { + if ( $query =3D=3D=3D 'title' || $query =3D=3D=3D 'curid' ) { + continue; // note: curid sets title + // Normal page view in query form can have action=3Dview. + } elseif ( $query =3D=3D=3D 'action' && in_array( $val, self::cacheable= PageActions() ) ) { + continue; + // Below are header setting params + } elseif ( $query =3D=3D=3D 'maxage' || $query =3D=3D=3D 'smaxage' ) { + continue; + } + return false; + } + $user =3D $context->getUser(); + // Check for non-standard user language; this covers uselang, + // and extensions for auto-detecting user language. + $ulang =3D $context->getLanguage()->getCode(); + $clang =3D $wgContLang->getCode(); + // Check that there are no other sources of variation + return !$user->getId() && !$user->getNewtalk() && $ulang =3D=3D $clang; + } + + /** + * Read from cache to context output + * @param $context IContextSource + * @return void + */ + public function loadFromFileCache( IContextSource $context ) { + global $wgMimeType, $wgLanguageCode; + + wfDebug( __METHOD__ . "()\n"); + $filename =3D $this->cachePath(); + $context->getOutput()->sendCacheControl(); + header( "Content-Type: $wgMimeType; charset=3DUTF-8" ); + header( "Content-Language: $wgLanguageCode" ); + if ( $this->useGzip() ) { + if ( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + } else { + /* Send uncompressed */ + readgzfile( $filename ); + return; + } + } + readfile( $filename ); + $context->getOutput()->disable(); // tell $wgOut that output is taken ca= re of + } + + /** + * Save this cache object with the given text. + * Use this as an ob_start() handler. + * @param $text string + * @return bool Whether $wgUseFileCache is enabled + */ + public function saveToFileCache( $text ) { + global $wgUseFileCache; + + if ( !$wgUseFileCache || strlen( $text ) < 512 ) { + // Disabled or empty/broken output (OOM and PHP errors) + return $text; + } + + wfDebug( __METHOD__ . "()\n", false); + + $now =3D wfTimestampNow(); + if ( $this->useGzip() ) { + $text =3D str_replace( + '', '\n", $text ); + } else { + $text =3D str_replace( + '', '\n", $text ); + } + + // Store text to FS... + $compressed =3D $this->saveText( $text ); + if ( $compressed =3D=3D=3D false ) { + return $text; // error + } + + // gzip output to buffer as needed and set headers... + if ( $this->useGzip() ) { + // @TODO: ugly wfClientAcceptsGzip() function - use context! + if ( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + return $compressed; + } else { + return $text; + } + } else { + return $text; + } + } + + /** + * Clear the file caches for a page for all actions + * @param $title Title + * @return bool Whether $wgUseFileCache is enabled + */ + public static function clearFileCache( Title $title ) { + global $wgUseFileCache; + + if ( !$wgUseFileCache ) { + return false; + } + + foreach ( self::cacheablePageActions() as $type ) { + $fc =3D self::newFromTitle( $title, $type ); + $fc->clearCache(); + } + + return true; + } +} diff --git a/includes/cache/LinkBatch.php b/includes/cache/LinkBatch.php new file mode 100644 index 0000000..17e8739 --- /dev/null +++ b/includes/cache/LinkBatch.php @@ -0,0 +1,207 @@ +addObj( $item ); + } + } + + /** + * Use ->setCaller( __METHOD__ ) to indicate which code is using this + * class. Only used in debugging output. + * @since 1.17 + * + * @param $caller + */ + public function setCaller( $caller ) { + $this->caller =3D $caller; + } + + /** + * @param $title Title + */ + public function addObj( $title ) { + if ( is_object( $title ) ) { + $this->add( $title->getNamespace(), $title->getDBkey() ); + } else { + wfDebug( "Warning: LinkBatch::addObj got invalid title object\n" ); + } + } + + public function add( $ns, $dbkey ) { + if ( $ns < 0 ) { + return; + } + if ( !array_key_exists( $ns, $this->data ) ) { + $this->data[$ns] =3D array(); + } + + $this->data[$ns][str_replace( ' ', '_', $dbkey )] =3D 1; + } + + /** + * Set the link list to a given 2-d array + * First key is the namespace, second is the DB key, value arbitrary + * + * @param $array array + */ + public function setArray( $array ) { + $this->data =3D $array; + } + + /** + * Returns true if no pages have been added, false otherwise. + * + * @return bool + */ + public function isEmpty() { + return ($this->getSize() =3D=3D 0); + } + + /** + * Returns the size of the batch. + * + * @return int + */ + public function getSize() { + return count( $this->data ); + } + + /** + * Do the query and add the results to the LinkCache object + * + * @return Array mapping PDBK to ID + */ + public function execute() { + $linkCache =3D LinkCache::singleton(); + return $this->executeInto( $linkCache ); + } + + /** + * Do the query and add the results to a given LinkCache object + * Return an array mapping PDBK to ID + * + * @param $cache LinkCache + * @return Array remaining IDs + */ + protected function executeInto( &$cache ) { + wfProfileIn( __METHOD__ ); + $res =3D $this->doQuery(); + $this->doGenderQuery(); + $ids =3D $this->addResultToCache( $cache, $res ); + wfProfileOut( __METHOD__ ); + return $ids; + } + + /** + * Add a ResultWrapper containing IDs and titles to a LinkCache object. + * As normal, titles will go into the static Title cache field. + * This function *also* stores extra fields of the title used for link + * parsing to avoid extra DB queries. + * + * @param $cache LinkCache + * @param $res + * @return Array of remaining titles + */ + public function addResultToCache( $cache, $res ) { + if ( !$res ) { + return array(); + } + + // For each returned entry, add it to the list of good links, and remove= it from $remaining + + $ids =3D array(); + $remaining =3D $this->data; + foreach ( $res as $row ) { + $title =3D Title::makeTitle( $row->page_namespace, $row->page_title ); + $cache->addGoodLinkObjFromRow( $title, $row ); + $ids[$title->getPrefixedDBkey()] =3D $row->page_id; + unset( $remaining[$row->page_namespace][$row->page_title] ); + } + + // The remaining links in $data are bad links, register them as such + foreach ( $remaining as $ns =3D> $dbkeys ) { + foreach ( $dbkeys as $dbkey =3D> $unused ) { + $title =3D Title::makeTitle( $ns, $dbkey ); + $cache->addBadLinkObj( $title ); + $ids[$title->getPrefixedDBkey()] =3D 0; + } + } + return $ids; + } + + /** + * Perform the existence test query, return a ResultWrapper with page_id = fields + * @return Bool|ResultWrapper + */ + public function doQuery() { + if ( $this->isEmpty() ) { + return false; + } + wfProfileIn( __METHOD__ ); + + // This is similar to LinkHolderArray::replaceInternal + $dbr =3D wfGetDB( DB_SLAVE ); + $table =3D 'page'; + $fields =3D array( 'page_id', 'page_namespace', 'page_title', 'page_len', + 'page_is_redirect', 'page_latest' ); + $conds =3D $this->constructSet( 'page', $dbr ); + + // Do query + $caller =3D __METHOD__; + if ( strval( $this->caller ) !=3D=3D '' ) { + $caller .=3D " (for {$this->caller})"; + } + $res =3D $dbr->select( $table, $fields, $conds, $caller ); + wfProfileOut( __METHOD__ ); + return $res; + } + + /** + * Do (and cache) {{GENDER:...}} information for userpages in this LinkBa= tch + * + * @return bool whether the query was successful + */ + public function doGenderQuery() { + if ( $this->isEmpty() ) { + return false; + } + + global $wgContLang; + if ( !$wgContLang->needsGenderDistinction() ) { + return false; + } + + $genderCache =3D GenderCache::singleton(); + $genderCache->dolinkBatch( $this->data, $this->caller ); + return true; + } + + /** + * Construct a WHERE clause which will match all the given titles. + * + * @param $prefix String: the appropriate table's field name prefix ('pag= e', 'pl', etc) + * @param $db DatabaseBase object to use + * @return mixed string with SQL where clause fragment, or false if no it= ems. + */ + public function constructSet( $prefix, $db ) { + return $db->makeWhereFrom2d( $this->data, "{$prefix}_namespace", "{$pref= ix}_title" ); + } +} diff --git a/includes/cache/LinkCache.php b/includes/cache/LinkCache.php new file mode 100644 index 0000000..a73eaaa --- /dev/null +++ b/includes/cache/LinkCache.php @@ -0,0 +1,216 @@ +mForUpdate, $update ); + } + + /** + * @param $title + * @return array|int + */ + public function getGoodLinkID( $title ) { + if ( array_key_exists( $title, $this->mGoodLinks ) ) { + return $this->mGoodLinks[$title]; + } else { + return 0; + } + } + + /** + * Get a field of a title object from cache. + * If this link is not good, it will return NULL. + * @param $title Title + * @param $field String: ('length','redirect','revision') + * @return mixed + */ + public function getGoodLinkFieldObj( $title, $field ) { + $dbkey =3D $title->getPrefixedDbKey(); + if ( array_key_exists( $dbkey, $this->mGoodLinkFields ) ) { + return $this->mGoodLinkFields[$dbkey][$field]; + } else { + return null; + } + } + + /** + * @param $title + * @return bool + */ + public function isBadLink( $title ) { + return array_key_exists( $title, $this->mBadLinks ); + } + + /** + * Add a link for the title to the link cache + * + * @param $id Integer: page's ID + * @param $title Title object + * @param $len Integer: text's length + * @param $redir Integer: whether the page is a redirect + * @param $revision Integer: latest revision's ID + */ + public function addGoodLinkObj( $id, $title, $len =3D -1, $redir =3D null= , $revision =3D false ) { + $dbkey =3D $title->getPrefixedDbKey(); + $this->mGoodLinks[$dbkey] =3D intval( $id ); + $this->mGoodLinkFields[$dbkey] =3D array( + 'length' =3D> intval( $len ), + 'redirect' =3D> intval( $redir ), + 'revision' =3D> intval( $revision ) ); + } + + /** + * Same as above with better interface. + * @since 1.19 + * @param $title Title + * @param $row object which has the fields page_id, page_is_redirect, + * page_latest + */ + public function addGoodLinkObjFromRow( $title, $row ) { + $dbkey =3D $title->getPrefixedDbKey(); + $this->mGoodLinks[$dbkey] =3D intval( $row->page_id ); + $this->mGoodLinkFields[$dbkey] =3D array( + 'length' =3D> intval( $row->page_len ), + 'redirect' =3D> intval( $row->page_is_redirect ), + 'revision' =3D> intval( $row->page_latest ), + ); + } + + /** + * @param $title Title + */ + public function addBadLinkObj( $title ) { + $dbkey =3D $title->getPrefixedDbKey(); + if ( !$this->isBadLink( $dbkey ) ) { + $this->mBadLinks[$dbkey] =3D 1; + } + } + + public function clearBadLink( $title ) { + unset( $this->mBadLinks[$title] ); + } + + /** + * @param $title Title + */ + public function clearLink( $title ) { + $dbkey =3D $title->getPrefixedDbKey(); + unset( $this->mBadLinks[$dbkey] ); + unset( $this->mGoodLinks[$dbkey] ); + unset( $this->mGoodLinkFields[$dbkey] ); + } + + public function getGoodLinks() { return $this->mGoodLinks; } + public function getBadLinks() { return array_keys( $this->mBadLinks ); } + + /** + * Add a title to the link cache, return the page_id or zero if non-exist= ent + * + * @param $title String: title to add + * @return Integer + */ + public function addLink( $title ) { + $nt =3D Title::newFromDBkey( $title ); + if( $nt ) { + return $this->addLinkObj( $nt ); + } else { + return 0; + } + } + + /** + * Add a title to the link cache, return the page_id or zero if non-exist= ent + * + * @param $nt Title object to add + * @return Integer + */ + public function addLinkObj( $nt ) { + global $wgAntiLockFlags; + wfProfileIn( __METHOD__ ); + + $key =3D $nt->getPrefixedDBkey(); + if ( $this->isBadLink( $key ) || $nt->isExternal() ) { + wfProfileOut( __METHOD__ ); + return 0; + } + $id =3D $this->getGoodLinkID( $key ); + if ( $id !=3D 0 ) { + wfProfileOut( __METHOD__ ); + return $id; + } + + if ( $key =3D=3D=3D '' ) { + wfProfileOut( __METHOD__ ); + return 0; + } + + # Some fields heavily used for linking... + if ( $this->mForUpdate ) { + $db =3D wfGetDB( DB_MASTER ); + if ( !( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) ) { + $options =3D array( 'FOR UPDATE' ); + } else { + $options =3D array(); + } + } else { + $db =3D wfGetDB( DB_SLAVE ); + $options =3D array(); + } + + $s =3D $db->selectRow( 'page', + array( 'page_id', 'page_len', 'page_is_redirect', 'page_latest' ), + array( 'page_namespace' =3D> $nt->getNamespace(), 'page_title' =3D> $nt= ->getDBkey() ), + __METHOD__, $options ); + # Set fields... + if ( $s !=3D=3D false ) { + $this->addGoodLinkObjFromRow( $nt, $s ); + $id =3D intval( $s->page_id ); + } else { + $this->addBadLinkObj( $nt ); + $id =3D 0; + } + + wfProfileOut( __METHOD__ ); + return $id; + } + + /** + * Clears cache + */ + public function clear() { + $this->mGoodLinks =3D array(); + $this->mGoodLinkFields =3D array(); + $this->mBadLinks =3D array(); + } +} diff --git a/includes/cache/MemcachedSessions.php b/includes/cache/Memcache= dSessions.php new file mode 100644 index 0000000..3673359 --- /dev/null +++ b/includes/cache/MemcachedSessions.php @@ -0,0 +1,98 @@ +get( memsess_key( $id ) ); + if( ! $data ) return ''; + return $data; +} + +/** + * Callback when writing session data. + * + * @param $id String: session id + * @param $data Mixed: session data + * @return Boolean: success + */ +function memsess_write( $id, $data ) { + global $wgMemc; + $wgMemc->set( memsess_key( $id ), $data, 3600 ); + return true; +} + +/** + * Callback to destroy a session when calling session_destroy(). + * + * @param $id String: session id + * @return Boolean: success + */ +function memsess_destroy( $id ) { + global $wgMemc; + + $wgMemc->delete( memsess_key( $id ) ); + return true; +} + +/** + * Callback to execute garbage collection. + * NOP: Memcached performs garbage collection. + * + * @param $maxlifetime Integer: maximum session life time + * @return Boolean: success + */ +function memsess_gc( $maxlifetime ) { + return true; +} + +function memsess_write_close() { + session_write_close(); +} + diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.= php new file mode 100644 index 0000000..146edd2 --- /dev/null +++ b/includes/cache/MessageCache.php @@ -0,0 +1,999 @@ +mMemc =3D $memCached; + $this->mDisable =3D !$useDB; + $this->mExpiry =3D $expiry; + } + + /** + * ParserOptions is lazy initialised. + * + * @return ParserOptions + */ + function getParserOptions() { + if ( !$this->mParserOptions ) { + $this->mParserOptions =3D new ParserOptions; + } + return $this->mParserOptions; + } + + /** + * Try to load the cache from a local file. + * Actual format of the file depends on the $wgLocalMessageCacheSerialized + * setting. + * + * @param $hash String: the hash of contents, to check validity. + * @param $code Mixed: Optional language code, see documenation of load(). + * @return false on failure. + */ + function loadFromLocal( $hash, $code ) { + global $wgCacheDirectory, $wgLocalMessageCacheSerialized; + + $filename =3D "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; + + # Check file existence + wfSuppressWarnings(); + $file =3D fopen( $filename, 'r' ); + wfRestoreWarnings(); + if ( !$file ) { + return false; // No cache file + } + + if ( $wgLocalMessageCacheSerialized ) { + // Check to see if the file has the hash specified + $localHash =3D fread( $file, 32 ); + if ( $hash =3D=3D=3D $localHash ) { + // All good, get the rest of it + $serialized =3D ''; + while ( !feof( $file ) ) { + $serialized .=3D fread( $file, 100000 ); + } + fclose( $file ); + return $this->setCache( unserialize( $serialized ), $code ); + } else { + fclose( $file ); + return false; // Wrong hash + } + } else { + $localHash =3D substr( fread( $file, 40 ), 8 ); + fclose( $file ); + if ( $hash !=3D $localHash ) { + return false; // Wrong hash + } + + # Require overwrites the member variable or just shadows it? + require( $filename ); + return $this->setCache( $this->mCache, $code ); + } + } + + /** + * Save the cache to a local file. + */ + function saveToLocal( $serialized, $hash, $code ) { + global $wgCacheDirectory; + + $filename =3D "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; + wfMkdirParents( $wgCacheDirectory, null, __METHOD__ ); // might fail + + wfSuppressWarnings(); + $file =3D fopen( $filename, 'w' ); + wfRestoreWarnings(); + + if ( !$file ) { + wfDebug( "Unable to open local cache file for writing\n" ); + return; + } + + fwrite( $file, $hash . $serialized ); + fclose( $file ); + wfSuppressWarnings(); + chmod( $filename, 0666 ); + wfRestoreWarnings(); + } + + function saveToScript( $array, $hash, $code ) { + global $wgCacheDirectory; + + $filename =3D "$wgCacheDirectory/messages-" . wfWikiID() . "-$code"; + $tempFilename =3D $filename . '.tmp'; + wfMkdirParents( $wgCacheDirectory, null, __METHOD__ ); // might fail + + wfSuppressWarnings(); + $file =3D fopen( $tempFilename, 'w' ); + wfRestoreWarnings(); + + if ( !$file ) { + wfDebug( "Unable to open local cache file for writing\n" ); + return; + } + + fwrite( $file, "mCache =3D array(" ); + + foreach ( $array as $key =3D> $message ) { + $key =3D $this->escapeForScript( $key ); + $message =3D $this->escapeForScript( $message ); + fwrite( $file, "'$key' =3D> '$message',\n" ); + } + + fwrite( $file, ");\n?>" ); + fclose( $file); + rename( $tempFilename, $filename ); + } + + function escapeForScript( $string ) { + $string =3D str_replace( '\\', '\\\\', $string ); + $string =3D str_replace( '\'', '\\\'', $string ); + return $string; + } + + /** + * Set the cache to $cache, if it is valid. Otherwise set the cache to fa= lse. + * + * @return bool + */ + function setCache( $cache, $code ) { + if ( isset( $cache['VERSION'] ) && $cache['VERSION'] =3D=3D MSG_CACHE_VE= RSION ) { + $this->mCache[$code] =3D $cache; + return true; + } else { + return false; + } + } + + /** + * Loads messages from caches or from database in this order: + * (1) local message cache (if $wgUseLocalMessageCache is enabled) + * (2) memcached + * (3) from the database. + * + * When succesfully loading from (2) or (3), all higher level caches are + * updated for the newest version. + * + * Nothing is loaded if member variable mDisable is true, either manually + * set by calling code or if message loading fails (is this possible?). + * + * Returns true if cache is already populated or it was succesfully popul= ated, + * or false if populating empty cache fails. Also returns true if Message= Cache + * is disabled. + * + * @param $code String: language to which load messages + */ + function load( $code =3D false ) { + global $wgUseLocalMessageCache; + + if( !is_string( $code ) ) { + # This isn't really nice, so at least make a note about it and try to + # fall back + wfDebug( __METHOD__ . " called without providing a language code\n" ); + $code =3D 'en'; + } + + # Don't do double loading... + if ( isset( $this->mLoadedLanguages[$code] ) ) { + return true; + } + + # 8 lines of code just to say (once) that message cache is disabled + if ( $this->mDisable ) { + static $shownDisabled =3D false; + if ( !$shownDisabled ) { + wfDebug( __METHOD__ . ": disabled\n" ); + $shownDisabled =3D true; + } + return true; + } + + # Loading code starts + wfProfileIn( __METHOD__ ); + $success =3D false; # Keep track of success + $where =3D array(); # Debug info, delayed to avoid spamming debug log to= o much + $cacheKey =3D wfMemcKey( 'messages', $code ); # Key in memc for messages + + # (1) local cache + # Hash of the contents is stored in memcache, to detect if local cache g= oes + # out of date (due to update in other thread?) + if ( $wgUseLocalMessageCache ) { + wfProfileIn( __METHOD__ . '-fromlocal' ); + + $hash =3D $this->mMemc->get( wfMemcKey( 'messages', $code, 'hash' ) ); + if ( $hash ) { + $success =3D $this->loadFromLocal( $hash, $code ); + if ( $success ) $where[] =3D 'got from local cache'; + } + wfProfileOut( __METHOD__ . '-fromlocal' ); + } + + # (2) memcache + # Fails if nothing in cache, or in the wrong version. + if ( !$success ) { + wfProfileIn( __METHOD__ . '-fromcache' ); + $cache =3D $this->mMemc->get( $cacheKey ); + $success =3D $this->setCache( $cache, $code ); + if ( $success ) { + $where[] =3D 'got from global cache'; + $this->saveToCaches( $cache, false, $code ); + } + wfProfileOut( __METHOD__ . '-fromcache' ); + } + + # (3) + # Nothing in caches... so we need create one and store it in caches + if ( !$success ) { + $where[] =3D 'cache is empty'; + $where[] =3D 'loading from database'; + + $this->lock( $cacheKey ); + + # Limit the concurrency of loadFromDB to a single process + # This prevents the site from going down when the cache expires + $statusKey =3D wfMemcKey( 'messages', $code, 'status' ); + $success =3D $this->mMemc->add( $statusKey, 'loading', MSG_LOAD_TIMEOUT= ); + if ( $success ) { + $cache =3D $this->loadFromDB( $code ); + $success =3D $this->setCache( $cache, $code ); + } + if ( $success ) { + $success =3D $this->saveToCaches( $cache, true, $code ); + if ( $success ) { + $this->mMemc->delete( $statusKey ); + } else { + $this->mMemc->set( $statusKey, 'error', 60 * 5 ); + wfDebug( "MemCached set error in MessageCache: restart memcached serv= er!\n" ); + } + } + $this->unlock($cacheKey); + } + + if ( !$success ) { + # Bad luck... this should not happen + $where[] =3D 'loading FAILED - cache is disabled'; + $info =3D implode( ', ', $where ); + wfDebug( __METHOD__ . ": Loading $code... $info\n" ); + $this->mDisable =3D true; + $this->mCache =3D false; + } else { + # All good, just record the success + $info =3D implode( ', ', $where ); + wfDebug( __METHOD__ . ": Loading $code... $info\n" ); + $this->mLoadedLanguages[$code] =3D true; + } + wfProfileOut( __METHOD__ ); + return $success; + } + + /** + * Loads cacheable messages from the database. Messages bigger than + * $wgMaxMsgCacheEntrySize are assigned a special value, and are loaded + * on-demand from the database later. + * + * @param $code String: language code. + * @return Array: loaded messages for storing in caches. + */ + function loadFromDB( $code ) { + wfProfileIn( __METHOD__ ); + global $wgMaxMsgCacheEntrySize, $wgLanguageCode, $wgAdaptiveMessageCache; + $dbr =3D wfGetDB( DB_SLAVE ); + $cache =3D array(); + + # Common conditions + $conds =3D array( + 'page_is_redirect' =3D> 0, + 'page_namespace' =3D> NS_MEDIAWIKI, + ); + + $mostused =3D array(); + if ( $wgAdaptiveMessageCache ) { + $mostused =3D $this->getMostUsedMessages(); + if ( $code !=3D=3D $wgLanguageCode ) { + foreach ( $mostused as $key =3D> $value ) { + $mostused[$key] =3D "$value/$code"; + } + } + } + + if ( count( $mostused ) ) { + $conds['page_title'] =3D $mostused; + } elseif ( $code !=3D=3D $wgLanguageCode ) { + $conds[] =3D 'page_title' . $dbr->buildLike( $dbr->anyString(), "/$code= " ); + } else { + # Effectively disallows use of '/' character in NS_MEDIAWIKI for uses + # other than language code. + $conds[] =3D 'page_title NOT' . $dbr->buildLike( $dbr->anyString(), '/'= , $dbr->anyString() ); + } + + # Conditions to fetch oversized pages to ignore them + $bigConds =3D $conds; + $bigConds[] =3D 'page_len > ' . intval( $wgMaxMsgCacheEntrySize ); + + # Load titles for all oversized pages in the MediaWiki namespace + $res =3D $dbr->select( 'page', 'page_title', $bigConds, __METHOD__ . "($= code)-big" ); + foreach ( $res as $row ) { + $cache[$row->page_title] =3D '!TOO BIG'; + } + + # Conditions to load the remaining pages with their contents + $smallConds =3D $conds; + $smallConds[] =3D 'page_latest=3Drev_id'; + $smallConds[] =3D 'rev_text_id=3Dold_id'; + $smallConds[] =3D 'page_len <=3D ' . intval( $wgMaxMsgCacheEntrySize ); + + $res =3D $dbr->select( + array( 'page', 'revision', 'text' ), + array( 'page_title', 'old_text', 'old_flags' ), + $smallConds, + __METHOD__ . "($code)-small" + ); + + foreach ( $res as $row ) { + $text =3D Revision::getRevisionText( $row ); + if( $text =3D=3D=3D false ) { + // Failed to fetch data; possible ES errors? + // Store a marker to fetch on-demand as a workaround... + $entry =3D '!TOO BIG'; + wfDebugLog( 'MessageCache', __METHOD__ . ": failed to load message pag= e text for {$row->page_title} ($code)" ); + } else { + $entry =3D ' ' . $text; + } + $cache[$row->page_title] =3D $entry; + } + + foreach ( $mostused as $key ) { + if ( !isset( $cache[$key] ) ) { + $cache[$key] =3D '!NONEXISTENT'; + } + } + + $cache['VERSION'] =3D MSG_CACHE_VERSION; + wfProfileOut( __METHOD__ ); + return $cache; + } + + /** + * Updates cache as necessary when message page is changed + * + * @param $title String: name of the page changed. + * @param $text Mixed: new contents of the page. + */ + public function replace( $title, $text ) { + global $wgMaxMsgCacheEntrySize; + wfProfileIn( __METHOD__ ); + + if ( $this->mDisable ) { + wfProfileOut( __METHOD__ ); + return; + } + + list( $msg, $code ) =3D $this->figureMessage( $title ); + + $cacheKey =3D wfMemcKey( 'messages', $code ); + $this->load( $code ); + $this->lock( $cacheKey ); + + $titleKey =3D wfMemcKey( 'messages', 'individual', $title ); + + if ( $text =3D=3D=3D false ) { + # Article was deleted + $this->mCache[$code][$title] =3D '!NONEXISTENT'; + $this->mMemc->delete( $titleKey ); + } elseif ( strlen( $text ) > $wgMaxMsgCacheEntrySize ) { + # Check for size + $this->mCache[$code][$title] =3D '!TOO BIG'; + $this->mMemc->set( $titleKey, ' ' . $text, $this->mExpiry ); + } else { + $this->mCache[$code][$title] =3D ' ' . $text; + $this->mMemc->delete( $titleKey ); + } + + # Update caches + $this->saveToCaches( $this->mCache[$code], true, $code ); + $this->unlock( $cacheKey ); + + // Also delete cached sidebar... just in case it is affected + $codes =3D array( $code ); + if ( $code =3D=3D=3D 'en' ) { + // Delete all sidebars, like for example on action=3Dpurge on the + // sidebar messages + $codes =3D array_keys( Language::getLanguageNames() ); + } + + global $wgMemc; + foreach ( $codes as $code ) { + $sidebarKey =3D wfMemcKey( 'sidebar', $code ); + $wgMemc->delete( $sidebarKey ); + } + + // Update the message in the message blob store + global $wgContLang; + MessageBlobStore::updateMessage( $wgContLang->lcfirst( $msg ) ); + + wfRunHooks( 'MessageCacheReplace', array( $title, $text ) ); + + wfProfileOut( __METHOD__ ); + } + + /** + * Shortcut to update caches. + * + * @param $cache Array: cached messages with a version. + * @param $memc Bool: Wether to update or not memcache. + * @param $code String: Language code. + * @return False on somekind of error. + */ + protected function saveToCaches( $cache, $memc =3D true, $code =3D false = ) { + wfProfileIn( __METHOD__ ); + global $wgUseLocalMessageCache, $wgLocalMessageCacheSerialized; + + $cacheKey =3D wfMemcKey( 'messages', $code ); + + if ( $memc ) { + $success =3D $this->mMemc->set( $cacheKey, $cache, $this->mExpiry ); + } else { + $success =3D true; + } + + # Save to local cache + if ( $wgUseLocalMessageCache ) { + $serialized =3D serialize( $cache ); + $hash =3D md5( $serialized ); + $this->mMemc->set( wfMemcKey( 'messages', $code, 'hash' ), $hash, $this= ->mExpiry ); + if ($wgLocalMessageCacheSerialized) { + $this->saveToLocal( $serialized, $hash, $code ); + } else { + $this->saveToScript( $cache, $hash, $code ); + } + } + + wfProfileOut( __METHOD__ ); + return $success; + } + + /** + * Represents a write lock on the messages key + * + * @param $key string + * + * @return Boolean: success + */ + function lock( $key ) { + $lockKey =3D $key . ':lock'; + for ( $i =3D 0; $i < MSG_WAIT_TIMEOUT && !$this->mMemc->add( $lockKey, 1= , MSG_LOCK_TIMEOUT ); $i++ ) { + sleep( 1 ); + } + + return $i >=3D MSG_WAIT_TIMEOUT; + } + + function unlock( $key ) { + $lockKey =3D $key . ':lock'; + $this->mMemc->delete( $lockKey ); + } + + /** + * Get a message from either the content language or the user language. + * + * @param $key String: the message cache key + * @param $useDB Boolean: get the message from the DB, false to use only + * the localisation + * @param $langcode String: code of the language to get the message for, = if + * it is a valid code create a language for that languag= e, + * if it is a string but not a valid code then make a ba= sic + * language object, if it is a false boolean then use the + * current users language (as a fallback for the old + * parameter functionality), or if it is a true boolean + * then use the wikis content language (also as a + * fallback). + * @param $isFullKey Boolean: specifies whether $key is a two part key + * "msg/lang". + * + * @return string|false + */ + function get( $key, $useDB =3D true, $langcode =3D true, $isFullKey =3D f= alse ) { + global $wgLanguageCode, $wgContLang; + + if ( is_int( $key ) ) { + // "Non-string key given" exception sometimes happens for numerical str= ings that become ints somewhere on their way here + $key =3D strval( $key ); + } + + if ( !is_string( $key ) ) { + throw new MWException( 'Non-string key given' ); + } + + if ( strval( $key ) =3D=3D=3D '' ) { + # Shortcut: the empty key is always missing + return false; + } + + $lang =3D wfGetLangObj( $langcode ); + if ( !$lang ) { + throw new MWException( "Bad lang code $langcode given" ); + } + + $langcode =3D $lang->getCode(); + + $message =3D false; + + # Normalise title-case input (with some inlining) + $lckey =3D str_replace( ' ', '_', $key ); + if ( ord( $key ) < 128 ) { + $lckey[0] =3D strtolower( $lckey[0] ); + $uckey =3D ucfirst( $lckey ); + } else { + $lckey =3D $wgContLang->lcfirst( $lckey ); + $uckey =3D $wgContLang->ucfirst( $lckey ); + } + + /** + * Record each message request, but only once per request. + * This information is not used unless $wgAdaptiveMessageCache + * is enabled. + */ + $this->mRequestedMessages[$uckey] =3D true; + + # Try the MediaWiki namespace + if( !$this->mDisable && $useDB ) { + $title =3D $uckey; + if( !$isFullKey && ( $langcode !=3D $wgLanguageCode ) ) { + $title .=3D '/' . $langcode; + } + $message =3D $this->getMsgFromNamespace( $title, $langcode ); + } + + # Try the array in the language object + if ( $message =3D=3D=3D false ) { + $message =3D $lang->getMessage( $lckey ); + if ( is_null( $message ) ) { + $message =3D false; + } + } + + # Try the array of another language + if( $message =3D=3D=3D false ) { + $parts =3D explode( '/', $lckey ); + # We may get calls for things that are http-urls from sidebar + # Let's not load nonexistent languages for those + # They usually have more than one slash. + if ( count( $parts ) =3D=3D 2 && $parts[1] !=3D=3D '' ) { + $message =3D Language::getMessageFor( $parts[0], $parts[1] ); + if ( is_null( $message ) ) { + $message =3D false; + } + } + } + + # Is this a custom message? Try the default language in the db... + if( ( $message =3D=3D=3D false || $message =3D=3D=3D '-' ) && + !$this->mDisable && $useDB && + !$isFullKey && ( $langcode !=3D $wgLanguageCode ) ) { + $message =3D $this->getMsgFromNamespace( $uckey, $wgLanguageCode ); + } + + # Final fallback + if( $message =3D=3D=3D false ) { + return false; + } + + # Fix whitespace + $message =3D strtr( $message, + array( + # Fix for trailing whitespace, removed by textarea + ' ' =3D> ' ', + # Fix for NBSP, converted to space by firefox + ' ' =3D> "\xc2\xa0", + ' ' =3D> "\xc2\xa0", + ) ); + + return $message; + } + + /** + * Get a message from the MediaWiki namespace, with caching. The key must + * first be converted to two-part lang/msg form if necessary. + * + * @param $title String: Message cache key with initial uppercase letter. + * @param $code String: code denoting the language to try. + * + * @return string|false + */ + function getMsgFromNamespace( $title, $code ) { + global $wgAdaptiveMessageCache; + + $this->load( $code ); + if ( isset( $this->mCache[$code][$title] ) ) { + $entry =3D $this->mCache[$code][$title]; + if ( substr( $entry, 0, 1 ) =3D=3D=3D ' ' ) { + return substr( $entry, 1 ); + } elseif ( $entry =3D=3D=3D '!NONEXISTENT' ) { + return false; + } elseif( $entry =3D=3D=3D '!TOO BIG' ) { + // Fall through and try invididual message cache below + } + } else { + // XXX: This is not cached in process cache, should it? + $message =3D false; + wfRunHooks( 'MessagesPreLoad', array( $title, &$message ) ); + if ( $message !=3D=3D false ) { + return $message; + } + + /** + * If message cache is in normal mode, it is guaranteed + * (except bugs) that there is always entry (or placeholder) + * in the cache if message exists. Thus we can do minor + * performance improvement and return false early. + */ + if ( !$wgAdaptiveMessageCache ) { + return false; + } + } + + # Try the individual message cache + $titleKey =3D wfMemcKey( 'messages', 'individual', $title ); + $entry =3D $this->mMemc->get( $titleKey ); + if ( $entry ) { + if ( substr( $entry, 0, 1 ) =3D=3D=3D ' ' ) { + $this->mCache[$code][$title] =3D $entry; + return substr( $entry, 1 ); + } elseif ( $entry =3D=3D=3D '!NONEXISTENT' ) { + $this->mCache[$code][$title] =3D '!NONEXISTENT'; + return false; + } else { + # Corrupt/obsolete entry, delete it + $this->mMemc->delete( $titleKey ); + } + } + + # Try loading it from the database + $revision =3D Revision::newFromTitle( Title::makeTitle( NS_MEDIAWIKI, $t= itle ) ); + if ( $revision ) { + $message =3D $revision->getText(); + if ($message =3D=3D=3D false) { + // A possibly temporary loading failure. + wfDebugLog( 'MessageCache', __METHOD__ . ": failed to load message pag= e text for {$title->getDbKey()} ($code)" ); + } else { + $this->mCache[$code][$title] =3D ' ' . $message; + $this->mMemc->set( $titleKey, ' ' . $message, $this->mExpiry ); + } + } else { + $message =3D false; + $this->mCache[$code][$title] =3D '!NONEXISTENT'; + $this->mMemc->set( $titleKey, '!NONEXISTENT', $this->mExpiry ); + } + + return $message; + } + + /** + * @param $message string + * @param $interface bool + * @param $language + * @param $title Title + * @return string + */ + function transform( $message, $interface =3D false, $language =3D null, $= title =3D null ) { + // Avoid creating parser if nothing to transform + if( strpos( $message, '{{' ) =3D=3D=3D false ) { + return $message; + } + + if ( $this->mInParser ) { + return $message; + } + + $parser =3D $this->getParser(); + if ( $parser ) { + $popts =3D $this->getParserOptions(); + $popts->setInterfaceMessage( $interface ); + $popts->setTargetLanguage( $language ); + + $userlang =3D $popts->setUserLang( $language ); + $this->mInParser =3D true; + $message =3D $parser->transformMsg( $message, $popts, $title ); + $this->mInParser =3D false; + $popts->setUserLang( $userlang ); + } + return $message; + } + + /** + * @return Parser + */ + function getParser() { + global $wgParser, $wgParserConf; + if ( !$this->mParser && isset( $wgParser ) ) { + # Do some initialisation so that we don't have to do it twice + $wgParser->firstCallInit(); + # Clone it and store it + $class =3D $wgParserConf['class']; + if ( $class =3D=3D 'Parser_DiffTest' ) { + # Uncloneable + $this->mParser =3D new $class( $wgParserConf ); + } else { + $this->mParser =3D clone $wgParser; + } + } + return $this->mParser; + } + + /** + * @param $text string + * @param $title Title + * @param $linestart bool + * @param $interface bool + * @param $language + * @return ParserOutput + */ + public function parse( $text, $title =3D null, $linestart =3D true, $inte= rface =3D false, $language =3D null ) { + if ( $this->mInParser ) { + return htmlspecialchars( $text ); + } + + $parser =3D $this->getParser(); + $popts =3D $this->getParserOptions(); + $popts->setInterfaceMessage( $interface ); + $popts->setTargetLanguage( $language ); + + wfProfileIn( __METHOD__ ); + if ( !$title || !$title instanceof Title ) { + global $wgTitle; + $title =3D $wgTitle; + } + // Sometimes $wgTitle isn't set either... + if ( !$title ) { + # It's not uncommon having a null $wgTitle in scripts. See r80898 + # Create a ghost title in such case + $title =3D Title::newFromText( 'Dwimmerlaik' ); + } + + $this->mInParser =3D true; + $res =3D $parser->parse( $text, $title, $popts, $linestart ); + $this->mInParser =3D false; + + wfProfileOut( __METHOD__ ); + return $res; + } + + function disable() { + $this->mDisable =3D true; + } + + function enable() { + $this->mDisable =3D false; + } + + /** + * Clear all stored messages. Mainly used after a mass rebuild. + */ + function clear() { + $langs =3D Language::getLanguageNames( false ); + foreach ( array_keys($langs) as $code ) { + # Global cache + $this->mMemc->delete( wfMemcKey( 'messages', $code ) ); + # Invalidate all local caches + $this->mMemc->delete( wfMemcKey( 'messages', $code, 'hash' ) ); + } + $this->mLoadedLanguages =3D array(); + } + + /** + * @param $key + * @return array + */ + public function figureMessage( $key ) { + global $wgLanguageCode; + $pieces =3D explode( '/', $key ); + if( count( $pieces ) < 2 ) { + return array( $key, $wgLanguageCode ); + } + + $lang =3D array_pop( $pieces ); + $validCodes =3D Language::getLanguageNames(); + if( !array_key_exists( $lang, $validCodes ) ) { + return array( $key, $wgLanguageCode ); + } + + $message =3D implode( '/', $pieces ); + return array( $message, $lang ); + } + + public static function logMessages() { + wfProfileIn( __METHOD__ ); + global $wgAdaptiveMessageCache; + if ( !$wgAdaptiveMessageCache || !self::$instance instanceof MessageCach= e ) { + wfProfileOut( __METHOD__ ); + return; + } + + $cachekey =3D wfMemckey( 'message-profiling' ); + $cache =3D wfGetCache( CACHE_DB ); + $data =3D $cache->get( $cachekey ); + + if ( !$data ) { + $data =3D array(); + } + + $age =3D self::$mAdaptiveDataAge; + $filterDate =3D substr( wfTimestamp( TS_MW, time() - $age ), 0, 8 ); + foreach ( array_keys( $data ) as $key ) { + if ( $key < $filterDate ) { + unset( $data[$key] ); + } + } + + $index =3D substr( wfTimestampNow(), 0, 8 ); + if ( !isset( $data[$index] ) ) { + $data[$index] =3D array(); + } + + foreach ( self::$instance->mRequestedMessages as $message =3D> $_ ) { + if ( !isset( $data[$index][$message] ) ) { + $data[$index][$message] =3D 0; + } + $data[$index][$message]++; + } + + $cache->set( $cachekey, $data ); + wfProfileOut( __METHOD__ ); + } + + /** + * @return array + */ + public function getMostUsedMessages() { + wfProfileIn( __METHOD__ ); + $cachekey =3D wfMemcKey( 'message-profiling' ); + $cache =3D wfGetCache( CACHE_DB ); + $data =3D $cache->get( $cachekey ); + if ( !$data ) { + wfProfileOut( __METHOD__ ); + return array(); + } + + $list =3D array(); + + foreach( $data as $messages ) { + foreach( $messages as $message =3D> $count ) { + $key =3D $message; + if ( !isset( $list[$key] ) ) { + $list[$key] =3D 0; + } + $list[$key] +=3D $count; + } + } + + $max =3D max( $list ); + foreach ( $list as $message =3D> $count ) { + if ( $count < intval( $max * self::$mAdaptiveInclusionThreshold ) ) { + unset( $list[$message] ); + } + } + + wfProfileOut( __METHOD__ ); + return array_keys( $list ); + } + + /** + * Get all message keys stored in the message cache for a given language. + * If $code is the content language code, this will return all message ke= ys + * for which MediaWiki:msgkey exists. If $code is another language code, = this + * will ONLY return message keys for which MediaWiki:msgkey/$code exists. + * @param $code string + * @return array of message keys (strings) + */ + public function getAllMessageKeys( $code ) { + global $wgContLang; + $this->load( $code ); + if ( !isset( $this->mCache[$code] ) ) { + // Apparently load() failed + return null; + } + $cache =3D $this->mCache[$code]; // Copy the cache + unset( $cache['VERSION'] ); // Remove the VERSION key + $cache =3D array_diff( $cache, array( '!NONEXISTENT' ) ); // Remove any = !NONEXISTENT keys + // Keys may appear with a capital first letter. lcfirst them. + return array_map( array( $wgContLang, 'lcfirst' ), array_keys( $cache ) = ); + } +} diff --git a/includes/cache/ObjectFileCache.php b/includes/cache/ObjectFile= Cache.php new file mode 100644 index 0000000..3356f1f --- /dev/null +++ b/includes/cache/ObjectFileCache.php @@ -0,0 +1,30 @@ +mKey =3D (string)$key; + $cache->mType =3D (string)$type; + + return $cache; + } + + /** + * Get the base file cache directory + * @return string + */ + protected function cacheDirectory() { + return $this->baseCacheDirectory() . '/object'; + } +} diff --git a/includes/cache/ResourceFileCache.php b/includes/cache/Resource= FileCache.php new file mode 100644 index 0000000..e73fc2d --- /dev/null +++ b/includes/cache/ResourceFileCache.php @@ -0,0 +1,87 @@ +getOnly() =3D=3D=3D 'styles' ) { + $cache->mType =3D 'css'; + } else { + $cache->mType =3D 'js'; + } + $modules =3D array_unique( $context->getModules() ); // remove duplicates + sort( $modules ); // normalize the order (permutation =3D> combination) + $cache->mKey =3D sha1( $context->getHash() . implode( '|', $modules ) ); + if ( count( $modules ) =3D=3D 1 ) { + $cache->mCacheWorthy =3D true; // won't take up much space + } + + return $cache; + } + + /** + * Check if an RL request can be cached. + * Caller is responsible for checking if any modules are private. + * @param $context ResourceLoaderContext + * @return bool + */ + public static function useFileCache( ResourceLoaderContext $context ) { + global $wgUseFileCache, $wgDefaultSkin, $wgLanguageCode; + if ( !$wgUseFileCache ) { + return false; + } + // Get all query values + $queryVals =3D $context->getRequest()->getValues(); + foreach ( $queryVals as $query =3D> $val ) { + if ( $query =3D=3D=3D 'modules' || $query =3D=3D=3D 'version' || $query= =3D=3D=3D '*' ) { + continue; // note: &* added as IE fix + } elseif ( $query =3D=3D=3D 'skin' && $val =3D=3D=3D $wgDefaultSkin ) { + continue; + } elseif ( $query =3D=3D=3D 'lang' && $val =3D=3D=3D $wgLanguageCode ) { + continue; + } elseif ( $query =3D=3D=3D 'only' && in_array( $val, array( 'styles', = 'scripts' ) ) ) { + continue; + } elseif ( $query =3D=3D=3D 'debug' && $val =3D=3D=3D 'false' ) { + continue; + } + return false; + } + return true; // cacheable + } + + /** + * Get the base file cache directory + * @return string + */ + protected function cacheDirectory() { + return $this->baseCacheDirectory() . '/resources'; + } + + /** + * Item has many recent cache misses + * @return bool + */ + public function isCacheWorthy() { + if ( $this->mCacheWorthy =3D=3D=3D null ) { + $this->mCacheWorthy =3D ( + $this->isCached() || // even stale cache indicates it was cache worthy + $this->getMissesRecent() >=3D self::MISS_THRESHOLD // many misses + ); + } + return $this->mCacheWorthy; + } +} diff --git a/includes/cache/SquidUpdate.php b/includes/cache/SquidUpdate.php new file mode 100644 index 0000000..d47b5b5 --- /dev/null +++ b/includes/cache/SquidUpdate.php @@ -0,0 +1,226 @@ +mMaxTitles =3D $wgMaxSquidPurgeTitles; + } else { + $this->mMaxTitles =3D $maxTitles; + } + if ( count( $urlArr ) > $this->mMaxTitles ) { + $urlArr =3D array_slice( $urlArr, 0, $this->mMaxTitles ); + } + $this->urlArr =3D $urlArr; + } + + /** + * @param $title Title + * + * @return SquidUpdate + */ + static function newFromLinksTo( &$title ) { + global $wgMaxSquidPurgeTitles; + wfProfileIn( __METHOD__ ); + + # Get a list of URLs linking to this page + $dbr =3D wfGetDB( DB_SLAVE ); + $res =3D $dbr->select( array( 'links', 'page' ), + array( 'page_namespace', 'page_title' ), + array( + 'pl_namespace' =3D> $title->getNamespace(), + 'pl_title' =3D> $title->getDBkey(), + 'pl_from=3Dpage_id' ), + __METHOD__ ); + $blurlArr =3D $title->getSquidURLs(); + if ( $dbr->numRows( $res ) <=3D $wgMaxSquidPurgeTitles ) { + foreach ( $res as $BL ) { + $tobj =3D Title::makeTitle( $BL->page_namespace, $BL->page_title ) ; + $blurlArr[] =3D $tobj->getInternalURL(); + } + } + + wfProfileOut( __METHOD__ ); + return new SquidUpdate( $blurlArr ); + } + + /** + * Create a SquidUpdate from an array of Title objects, or a TitleArray o= bject + * + * @param $titles array + * @param $urlArr array + * + * @return SquidUpdate + */ + static function newFromTitles( $titles, $urlArr =3D array() ) { + global $wgMaxSquidPurgeTitles; + $i =3D 0; + foreach ( $titles as $title ) { + $urlArr[] =3D $title->getInternalURL(); + if ( $i++ > $wgMaxSquidPurgeTitles ) { + break; + } + } + return new SquidUpdate( $urlArr ); + } + + /** + * @param $title Title + * + * @return SquidUpdate + */ + static function newSimplePurge( &$title ) { + $urlArr =3D $title->getSquidURLs(); + return new SquidUpdate( $urlArr ); + } + + /** + * Purges the list of URLs passed to the constructor + */ + function doUpdate() { + SquidUpdate::purge( $this->urlArr ); + } + + /** + * Purges a list of Squids defined in $wgSquidServers. + * $urlArr should contain the full URLs to purge as values + * (example: $urlArr[] =3D 'http://my.host/something') + * XXX report broken Squids per mail or log + * + * @param $urlArr array + * @return void + */ + static function purge( $urlArr ) { + global $wgSquidServers, $wgHTCPMulticastAddress, $wgHTCPPort; + + /*if ( (@$wgSquidServers[0]) =3D=3D 'echo' ) { + echo implode("
\n", $urlArr) . "
\n"; + return; + }*/ + + if( !$urlArr ) { + return; + } + + if ( $wgHTCPMulticastAddress && $wgHTCPPort ) { + SquidUpdate::HTCPPurge( $urlArr ); + } + + wfProfileIn( __METHOD__ ); + + $maxSocketsPerSquid =3D 8; // socket cap per Squid + $urlsPerSocket =3D 400; // 400 seems to be a good tradeoff, opening a so= cket takes a while + $socketsPerSquid =3D ceil( count( $urlArr ) / $urlsPerSocket ); + if ( $socketsPerSquid > $maxSocketsPerSquid ) { + $socketsPerSquid =3D $maxSocketsPerSquid; + } + + $pool =3D new SquidPurgeClientPool; + $chunks =3D array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSq= uid ) ); + foreach ( $wgSquidServers as $server ) { + foreach ( $chunks as $chunk ) { + $client =3D new SquidPurgeClient( $server ); + foreach ( $chunk as $url ) { + $client->queuePurge( $url ); + } + $pool->addClient( $client ); + } + } + $pool->run(); + + wfProfileOut( __METHOD__ ); + } + + /** + * @throws MWException + * @param $urlArr array + */ + static function HTCPPurge( $urlArr ) { + global $wgHTCPMulticastAddress, $wgHTCPMulticastTTL, $wgHTCPPort; + wfProfileIn( __METHOD__ ); + + $htcpOpCLR =3D 4; // HTCP CLR + + // @todo FIXME: PHP doesn't support these socket constants (include/linu= x/in.h) + if( !defined( "IPPROTO_IP" ) ) { + define( "IPPROTO_IP", 0 ); + define( "IP_MULTICAST_LOOP", 34 ); + define( "IP_MULTICAST_TTL", 33 ); + } + + // pfsockopen doesn't work because we need set_sock_opt + $conn =3D socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); + if ( $conn ) { + // Set socket options + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 ); + if ( $wgHTCPMulticastTTL !=3D 1 ) + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL, + $wgHTCPMulticastTTL ); + + foreach ( $urlArr as $url ) { + if( !is_string( $url ) ) { + throw new MWException( 'Bad purge URL' ); + } + $url =3D SquidUpdate::expand( $url ); + + // Construct a minimal HTCP request diagram + // as per RFC 2756 + // Opcode 'CLR', no response desired, no auth + $htcpTransID =3D rand(); + + $htcpSpecifier =3D pack( 'na4na*na8n', + 4, 'HEAD', strlen( $url ), $url, + 8, 'HTTP/1.0', 0 ); + + $htcpDataLen =3D 8 + 2 + strlen( $htcpSpecifier ); + $htcpLen =3D 4 + $htcpDataLen + 2; + + // Note! Squid gets the bit order of the first + // word wrong, wrt the RFC. Apparently no other + // implementation exists, so adapt to Squid + $htcpPacket =3D pack( 'nxxnCxNxxa*n', + $htcpLen, $htcpDataLen, $htcpOpCLR, + $htcpTransID, $htcpSpecifier, 2); + + // Send out + wfDebug( "Purging URL $url via HTCP\n" ); + socket_sendto( $conn, $htcpPacket, $htcpLen, 0, + $wgHTCPMulticastAddress, $wgHTCPPort ); + } + } else { + $errstr =3D socket_strerror( socket_last_error() ); + wfDebug( __METHOD__ . "(): Error opening UDP socket: $errstr\n" ); + } + wfProfileOut( __METHOD__ ); + } + + /** + * Expand local URLs to fully-qualified URLs using the internal protocol + * and host defined in $wgInternalServer. Input that's already fully- + * qualified will be passed through unchanged. + * + * This is used to generate purge URLs that may be either local to the + * main wiki or include a non-native host, such as images hosted on a + * second internal server. + * + * Client functions should not need to call this. + * + * @param $url string + * + * @return string + */ + static function expand( $url ) { + return wfExpandUrl( $url, PROTO_INTERNAL ); + } +}