classes/search/PaperSearchIndex.inc.php

Go to the documentation of this file.

<?php

//$Id$

import('search.SearchFileParser');
import('search.SearchHTMLParser');
import('search.SearchHelperParser');
import('search.PaperSearch');

// Words are truncated to at most this length
define('SEARCH_KEYWORD_MAX_LENGTH', 40);

class PaperSearchIndex {

   function indexObjectKeywords($objectId, $text, &$position) {
      $searchDao =& DAORegistry::getDAO('PaperSearchDAO');
      $keywords =& PaperSearchIndex::filterKeywords($text);
      for ($i = 0, $count = count($keywords); $i < $count; $i++) {
         if ($searchDao->insertObjectKeyword($objectId, $keywords[$i], $position) !== null) {
            $position += 1;
         }
      }
   }

   function updateTextIndex($paperId, $type, $text, $assocId = null) {
         $searchDao =& DAORegistry::getDAO('PaperSearchDAO');
         $objectId = $searchDao->insertObject($paperId, $type, $assocId);
         $position = 0;
         PaperSearchIndex::indexObjectKeywords($objectId, $text, $position);
   }

   function updateFileIndex($paperId, $type, $fileId) {
      import('file.PaperFileManager');
      $fileMgr = new PaperFileManager($paperId);
      $file =& $fileMgr->getFile($fileId);

      if (isset($file)) {
         $parser =& SearchFileParser::fromFile($file);
      }

      if (isset($parser)) {
         if ($parser->open()) {
            $searchDao =& DAORegistry::getDAO('PaperSearchDAO');
            $objectId = $searchDao->insertObject($paperId, $type, $fileId);

            $position = 0;
            while(($text = $parser->read()) !== false) {
               PaperSearchIndex::indexObjectKeywords($objectId, $text, $position);
            }
            $parser->close();
         }
      }
   }

   function deleteTextIndex($paperId, $type = null, $assocId = null) {
      $searchDao =& DAORegistry::getDAO('PaperSearchDAO');
      return $searchDao->deletePaperKeywords($paperId, $type, $assocId);
   }

   function &filterKeywords($text, $allowWildcards = false) {
      $minLength = Config::getVar('search', 'min_word_length');
      $stopwords =& PaperSearchIndex::loadStopwords();

      // Join multiple lines into a single string
      if (is_array($text)) $text = join("\n", $text);

      $cleanText = Core::cleanVar($text);

      // Remove punctuation
      $cleanText = String::regexp_replace('/[!"\#\$%\'\(\)\.\?@\[\]\^`\{\}~]/', '', $cleanText);
      $cleanText = String::regexp_replace('/[\+,:;&\/<=>\|\\\]/', ' ', $cleanText);
      $cleanText = String::regexp_replace('/[\*]/', $allowWildcards ? '%' : ' ', $cleanText);
      $cleanText = String::strtolower($cleanText);

      // Split into words
      $words = preg_split('/\s+/', $cleanText);

      // FIXME Do not perform further filtering for some fields, e.g., author names?

      // Remove stopwords
      $keywords = array();
      foreach ($words as $k) {
         if (!isset($stopwords[$k]) && String::strlen($k) >= $minLength && !is_numeric($k)) {
            $keywords[] = String::substr($k, 0, SEARCH_KEYWORD_MAX_LENGTH);
         }
      }
      return $keywords;
   }

   function &loadStopwords() {
      static $searchStopwords;

      if (!isset($searchStopwords)) {
         // Load stopwords only once per request (FIXME Cache?)
         $searchStopwords = array_count_values(array_filter(file(Config::getVar('general', 'registry_dir') . '/stopwords.txt'), create_function('&$a', 'return ($a = trim($a)) && !empty($a) && $a[0] != \'#\';')));
         $searchStopwords[''] = 1;
      }

      return $searchStopwords;
   }

   function indexPaperMetadata(&$paper) {
      // Build author keywords
      $authorText = array();
      $authors = $paper->getAuthors();
      for ($i=0, $count=count($authors); $i < $count; $i++) {
         $author =& $authors[$i];
         array_push($authorText, $author->getFirstName());
         array_push($authorText, $author->getMiddleName());
         array_push($authorText, $author->getLastName());
         array_push($authorText, $author->getAffiliation());
         $bios = $author->getBiography(null);
         if (is_array($bios)) foreach ($bios as $bio) { // Localized
            array_push($authorText, strip_tags($bio));
         }
      }

      // Update search index
      $paperId = $paper->getId();
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_AUTHOR, $authorText);
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_TITLE, $paper->getTitle(null));

      $trackDao =& DAORegistry::getDAO('TrackDAO');
      $track =& $trackDao->getTrack($paper->getTrackId());
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_ABSTRACT, $paper->getAbstract(null));
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_DISCIPLINE, $paper->getDiscipline(null));
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_SUBJECT, array_merge(array_values((array) $paper->getSubjectClass(null)), array_values((array) $paper->getSubject(null))));
      PaperSearchIndex::updateTextIndex($paperId, PAPER_SEARCH_TYPE, $paper->getType(null));
      PaperSearchIndex::updateTextIndex(
         $paperId,
         PAPER_SEARCH_COVERAGE,
         array_merge(
            array_values((array) $paper->getCoverageGeo(null)),
            array_values((array) $paper->getCoverageChron(null)),
            array_values((array) $paper->getCoverageSample(null))
         )
      );
      // FIXME Index sponsors too?
   }

   function indexSuppFileMetadata(&$suppFile) {
      // Update search index
      $paperId = $suppFile->getPaperId();
      PaperSearchIndex::updateTextIndex(
         $paperId,
         PAPER_SEARCH_SUPPLEMENTARY_FILE,
         array_merge(
            array_values((array) $suppFile->getTitle(null)),
            array_values((array) $suppFile->getCreator(null)),
            array_values((array) $suppFile->getSubject(null)),
            array_values((array) $suppFile->getTypeOther(null)),
            array_values((array) $suppFile->getDescription(null)),
            array_values((array) $suppFile->getSource(null))
         ),
         $suppFile->getFileId()
      );
   }

   function indexPaperFiles(&$paper) {
      // Index supplementary files
      $fileDao =& DAORegistry::getDAO('SuppFileDAO');
      $files =& $fileDao->getSuppFilesByPaper($paper->getId());
      foreach ($files as $file) {
         if ($file->getFileId()) {
            PaperSearchIndex::updateFileIndex($paper->getId(), PAPER_SEARCH_SUPPLEMENTARY_FILE, $file->getFileId());
         }
         PaperSearchIndex::indexSuppFileMetadata($file);
      }
      unset($files);

      // Index galley files
      $fileDao =& DAORegistry::getDAO('PaperGalleyDAO');
      $files =& $fileDao->getGalleysByPaper($paper->getId());
      foreach ($files as $file) {
         if ($file->getFileId()) {
            PaperSearchIndex::updateFileIndex($paper->getId(), PAPER_SEARCH_GALLEY_FILE, $file->getFileId());
         }
      }
   }

   function rebuildIndex($log = false) {
      // Clear index
      if ($log) echo 'Clearing index ... ';
      $searchDao =& DAORegistry::getDAO('PaperSearchDAO');
      // FIXME Abstract into PaperSearchDAO?
      $searchDao->update('DELETE FROM paper_search_object_keywords');
      $searchDao->update('DELETE FROM paper_search_objects');
      $searchDao->update('DELETE FROM paper_search_keyword_list');
      $searchDao->setCacheDir(Config::getVar('files', 'files_dir') . '/_db');
      $searchDao->_dataSource->CacheFlush();
      if ($log) echo "done\n";

      // Build index
      $schedConfDao =& DAORegistry::getDAO('SchedConfDAO');
      $paperDao =& DAORegistry::getDAO('PaperDAO');

      $schedConfs =& $schedConfDao->getSchedConfs();
      while (!$schedConfs->eof()) {
         $schedConf =& $schedConfs->next();
         $numIndexed = 0;

         if ($log) echo "Indexing \"", $schedConf->getFullTitle(), "\" ... ";

         $papers =& $paperDao->getPapersBySchedConfId($schedConf->getId());
         while (!$papers->eof()) {
            $paper =& $papers->next();
            if ($paper->getDateSubmitted()) {
               PaperSearchIndex::indexPaperMetadata($paper);
               PaperSearchIndex::indexPaperFiles($paper);
               $numIndexed++;
            }
            unset($paper);
         }

         if ($log) echo $numIndexed, " papers indexed\n";
         unset($schedConf);
      }
   }

}

?>

---