datapoint_simplepie.module

<?php
// $Id$

/**
 * @file
 * Implementation of the SimplePie feed parser for the datapoint module
 */

//Possible future developments:
//Make the parser exchangeable

/**
 * Implements hook_datapoint_parser().
 */
function datapoint_simplepie_datapoint_parser() {
  return array(
    'name'                        => t('Feed Parser - SimplePie'),
    'source configuration'        => '_datapoint_simplepie_source_configuration',
    'source configuration submit' => '_datapoint_simplepie_source_configuration_submit',
    'fetch'                       => '_datapoint_simplepie_fetch',
    'cleanup'                     => '_datapoint_simplepie_cleanup',
    'update check'                => '_datapoint_simplepie_update_check',
  );
}

/**
 * Assigned in datapoint_simplepie_datapoint_parser() - adds some configurations
 */
function _datapoint_simplepie_source_configuration($configuration) {
  $settings = array();
  $settings['simplepie_feed_url'] = array(
    '#type'          => 'textfield',
    '#title'         => t('Feed source'),
    '#default_value' => $configuration['feed url'],
  );
  $settings['simplepie_update_limit'] = array(
    '#type'          => 'select',
    '#options' => drupal_map_assoc(array(1800, 3600, 21600, 43200, 86400, 604800, 2419200), 'format_interval'),
    '#title'         => t('Update limit'),
    '#default_value' => $configuration['update limit'],
  );
  $settings['simplepie_do_cleanup'] = array(
    '#type'          => 'checkbox',
    '#title'         => t("Don't do any cleanup"),
    '#default_value' => $configuration['no cleanup'],
  );
  return $settings;
}

/**
 * Assigned in datapoint_simplepie_datapoint_parser() - handles the data from the configuration
 */
function _datapoint_simplepie_source_configuration_submit($values) {
  return array(
    'feed url' => $values['simplepie_feed_url'],
    'update limit' => $values['simplepie_update_limit'],
    'no cleanup' => $values['simplepie_no_cleanup'],
  );
}

/**
 * Assigned in datapoint_simplepie_datapoint_parser() - handles if the feed should be updated
 */
function _datapoint_simplepie_update_check($since_last_update, $source) {
  $update_limit = isset($source['source configuration']['update limit']) ? $source['source configuration']['update limit'] : 60 * 30; // half on hour;
  if ($since_last_update > $update_limit) {
    return TRUE;
  }
}

function datapoint_simplepie_datapoint_cleanup() {
  $sources = db_query("SELECT sid, name, configuration FROM {datapoint_source} WHERE source_type = 'feedsimplepie'");
  while ($source = db_fetch_object($sources)) {
    $source->configuration = unserialize($source->configuration);
    // Check if the source is marked to be cleaned.
    if (!$source->configuration['source configuration']['no cleanup']) {
      $matches = array();
      $feedurl = $source->configuration['source configuration']['feed url'];
      if (preg_match('/\[(.+)\]/', $feedurl, $matches)) {
        $key = $matches[1];
        $iterators = _datapoint_simplepie_get_iterators();
        if (array_key_exists($key, $iterators)) {
          $removed = 0;
          $result = db_query("SELECT v.statevalue, COUNT(v.vid) AS amount FROM {datapoint} d INNER JOIN {datapoint_simplepie_vertex} v ON d.vid = v.vid WHERE d.sid = %d AND v.iterator = '%s' GROUP BY v.statevalue HAVING amount > 15", $source->sid, $key);
          while ($state = db_fetch_object($result)) {
            $result_to_remove = db_query_range("SELECT d.pid, d.nid FROM {datapoint} d INNER JOIN {datapoint_simplepie_vertex} v ON d.vid = v.vid WHERE d.sid = %d AND v.iterator = '%s' AND v.statevalue = %s ORDER BY v.published ASC", $source->sid, $key, $state->statevalue, 0, $state->amount - 10);

            $pids = array();
            while ($to_remove = db_fetch_object($result_to_remove)) {
              $pids[] = $to_remove->pid;
              node_delete($to_remove->nid);
            }
            $placeholders = db_placeholders($pids);
            db_query("DELETE FROM {datapoint} WHERE pid IN ($placeholders)", $pids);
            db_query("DELETE FROM {datapoint_simplepie_vertex} WHERE pid IN ($placeholders)", $pids);
            $removed = $removed + $state->amount - 10;
          }
          if ($removed) {
            watchdog('datapoint simplepie', 'Removed @count items from @url', array('@count' => $removed, '@url' => $feedurl));
          }
          return;
        }
      }

      $amount = db_result(db_query("SELECT COUNT(v.vid) FROM {datapoint} d INNER JOIN {datapoint_simplepie_vertex} v ON d.vid = v.vid WHERE d.sid = %d", $source->sid, $key));
      if ($amount > 10) {
        $result_to_remove = db_query_range("SELECT d.pid, d.nid FROM {datapoint} d INNER JOIN {datapoint_simplepie_vertex} v ON d.vid = v.vid WHERE d.sid = %d ORDER BY v.published ASC", $source->sid, 0, $amount - 10);

        $pids = array();
        while ($to_remove = db_fetch_object($result_to_remove)) {
          $pids[] = $to_remove->pid;
          node_delete($to_remove->nid);
        }

        $placeholders = db_placeholders($pids);
        db_query("DELETE FROM {datapoint} WHERE pid IN (' . $placeholders . ')", $pids);
        db_query("DELETE FROM {datapoint_simplepie_vertex} WHERE pid IN (' . $placeholders . ')", $pids);
      }
      watchdog('datapoint simplepie', 'Removed @count items from @url', array('@count' => $amount - 10, '@url' => $feedurl));
    }
  }
}

/**
 * Check for time limits in fetch processing.
 * 
 * @return
 *   Number of seconds left, zero if none.
 */
function _datapoint_simplepie_fetch_time() {
  static $time_limit;

  if (!$time_limit) {
    $max = ini_get('max_execution_time');

    if (!$max) {
      $max = 240;
    }

    $time_limit = time() + 0.15 * $max;

    // However, check for left time, maybe some other cron processing already occured
    $time_limit = min($time_limit, variable_get('cron_semaphore', time()) + $max);
  }
  return max($time_limit - time(), 0);
}

function _datapoint_simplepie_get_iterators() {
  static $iterators;

  if (!isset($iterators)) {
    //Returns all iterators and enables modules to alter it
    $iterators = module_invoke_all('datapoint_simplepie_iterators');
    drupal_alter('datapoint_simplepie_iterators', $iterators);
  }

  return $iterators;
}

/**
 * Assigned in datapoint_simplepie_datapoint_parser() - fetches the source
 */
function _datapoint_simplepie_fetch($source, &$result) {
  $feedurl = $source['source configuration']['feed url'];
  $matches = array();

  if (preg_match('/\[(.+)\]/', $feedurl, $matches)) {
    $key = $matches[1];
    $iterators = _datapoint_simplepie_get_iterators();
    if (array_key_exists($key, $iterators)) {
      $state = isset($source['state']) ? $source['state'] : NULL;
      $count = 0;

      while (_datapoint_simplepie_fetch_time() && $replace = call_user_func_array($iterators[$key], array(&$state))) {
        $currenturl = str_replace('[' . $key . ']', $replace, $feedurl);
        $force_refresh = isset($source['force refresh']) && $source['force refresh'];
        _datapoint_simplepie_grab_and_add($currenturl, $result, $key, $replace, $force_refresh);
        $count = $count + 1;
      }

      watchdog('datapoint simplepie', 'Fetched @count feeds from @url', array('@count' => $count, '@url' => $feedurl));

      if ($replace) {
        $sid = $source['sid'];
        unset($source['sid']);
        $source['state'] = $state;
        db_query("UPDATE {datapoint_source} SET configuration='%s' WHERE sid=%d", serialize($source), $sid);
        return FALSE;
      }
      else {
        $sid = $source['sid'];
        unset($source['sid'], $source['state']);
        db_query("UPDATE {datapoint_source} SET configuration='%s' WHERE sid=%d", serialize($source), $sid);
      }

      return;
    }
  }
  _datapoint_simplepie_grab_and_add($feedurl, $result);
  watchdog('datapoint simplepie', 'Fetched @url', array('@url' => $feedurl));
}

function _datapoint_simplepie_grab_and_add($feedurl, &$result, $iterator = NULL, $statevalue = NULL, $force_refresh = FALSE) {
  $parsed = _datapoint_simplepie_parse($feedurl, $force_refresh);

  if (empty($parsed->items)) {
    return;
  }

  foreach ($parsed->items as $item) {
    //Create/get result objects
    $point = $result->point(md5($item->options->guid));
    $vertex = new DatapointVertex($item->options->timestamp);

    //If the point was created now it needs to get a name
    $current_name = $point->getName();
    if (empty($current_name)) {
      $point->setName(html_entity_decode(strip_tags($item->title), ENT_QUOTES, 'UTF-8'));
    }

    if (!empty($item->description)) {
      $vertex->value('description', $item->description);
    }

    $vertex->value('url',        $item->options->original_url);
    $vertex->value('published',  $item->options->timestamp);
    $vertex->value('iterator',   $iterator);
    $vertex->value('statevalue', $statevalue);

    // GeoRSS
    if (!empty($item->position)) {
      $vertex->value('position', $item->position);
    }

    $point->addVertex($vertex);
  }
}

/**
 * Parsing the feed
 *
 * @param $url
 *  The feed's url
 * @return
 *  The structured datas extracted from the feed
 */
function _datapoint_simplepie_parse($url, $force_refresh = FALSE) {
  $parser = _datapoint_simplepie_get_parser($url, $force_refresh);
  if ($parser->error) {
    watchdog('datapoint simplepie', 'Simplepie parser error: !error', array('!error' => $parser->error));
    return FALSE;
  }
  // Do we have html_entity_decode? Some feeds return html entities in the links
  $entity_decode = function_exists('html_entity_decode');
  // Construct the standard form of the parsed feed
  $parsed_source = new stdClass();
  $parsed_source->description = $parser->get_description();
  $parsed_source->title = _datapoint_simplepie_title($parser->get_title(), $parser->get_description());
  $parsed_source->options = new stdClass();
  $parsed_source->options->link = $entity_decode ? html_entity_decode($parser->get_link()) : $parser->get_link();
  $parsed_source->items = array();
  $items_num = $parser->get_item_quantity();
  for ($i = 0; $i < $items_num; $i++) {
    $curr_item = new stdClass();
    $simplepie_item = $parser->get_item($i);
    $curr_item->title = _datapoint_simplepie_title($simplepie_item->get_title(), $simplepie_item->get_content());
    $curr_item->description = $simplepie_item->get_content();
    $curr_item->options = new stdClass();
    $curr_item->options->original_url = $entity_decode ? html_entity_decode($simplepie_item->get_link()) : $simplepie_item->get_link();
    // U = std. unix timestamp
    $curr_item->options->timestamp = $simplepie_item->get_date("U");
    $curr_item->options->guid = $simplepie_item->get_id();
    $curr_item->options->original_author = $simplepie_item->get_author();
    // Extract tags related to the item
    $simplepie_tags = $simplepie_item->get_categories();
    $tags = array();
    $domains = array();
    if (count($simplepie_tags) > 0) {
      foreach ($simplepie_tags as $tag) {
        $tags[] = (string) $tag->term;
        $domain = (string) $tag->get_scheme();
        if (!empty($domain)) {
          if (!isset($domains[$domain])) {
            $domains[$domain] = array();
          }
          $domains[$domain][] = count($tags) - 1;
        }
      }
    }
    $curr_item->options->domains = $domains;
    $curr_item->options->tags = $tags;
    // Stick the raw data onto the feed item.
    $curr_item->options->raw = $simplepie_item->data;
    // GeoRSS
    if ($lat = $simplepie_item->get_latitude()) {
      $curr_item->position = $lat . ' ' . $simplepie_item->get_longitude();
    }
    $parsed_source->items[] = $curr_item;
  }
  return $parsed_source;
}

/**
 * Set the default caching directory if the current setting is not useable
 */
function _datapoint_simplepie_sanitize_cache() {
  static $cache_location;

  if (isset($cache_location)) {
    return $cache_location;
  }

  $cache_location = file_directory_path() .'/simplepie_cache';
  if (!is_writeable($cache_location) || !is_dir($cache_location)) {
    $cache_location = file_create_path($cache_location);
    if (!file_exists($cache_location) && is_writable(file_directory_path())) {
      mkdir($cache_location);
    }
    if (!is_writeable($cache_location)) {
      return $cache_location = FALSE;
    }
  }
  return $cache_location;
}

/**
 * Set SimplePie setting
 * @param $url
 *  The feed's url
 * @return
 *  SimplePie object
 */
function _datapoint_simplepie_get_parser($url, $force_refresh = FALSE) {
  if (module_exists('parser_simplepie') && module_load_include('inc', 'parser_simplepie', 'simplepie') !== FALSE) {
    $cache_location = _parser_simplepie_sanitize_cache();
  }
  else {
    module_load_include('inc', 'datapoint_simplepie', 'simplepie');
    $cache_location = _datapoint_simplepie_sanitize_cache();
  }

  $parser = new SimplePie();
  $parser->set_feed_url($url);
  $parser->set_timeout(15);
  $parser->set_stupidly_fast(TRUE);
  $parser->encode_instead_of_strip(FALSE);
  $parser->enable_cache($cache_location !== FALSE ? TRUE : FALSE);
  if ($force_refresh) {
    $parser->set_cache_duration(0);
  }
  $parser->set_cache_location($cache_location);
  $parser->init();

  return $parser;
}

/**
 * Implementation of hook_views_api().
 */
function datapoint_simplepie_views_api() {
  return array(
    'api' => 2,
    'path' => drupal_get_path('module', 'datapoint_simplepie') . '/includes',
  );
}

/**
 * Prepare raw data to be a title
 * Straight copy of _parser_simplepie_title() from parser_simplepie.module.
 */
function _datapoint_simplepie_title($title, $body = FALSE) {
    if (empty($title) && !empty($body)) {
    // Explode to words and use the first 3 words.
    $words = preg_split("/[\s,]+/", $body);
    $title = $words[0] .' '. $words[1] .' '. $words[2];
  }
  if (version_compare(PHP_VERSION, '5.0.0', '<')) {
    return html_entity_decode(strip_tags($title), ENT_QUOTES);
  }
  else {
    return html_entity_decode(strip_tags($title), ENT_QUOTES, 'UTF-8');
  }
}