F/handle import outside of drupal cron

README.md

@@ -25,6 +25,11 @@ See https://github.com/syddjurs/os2web/wiki for further instructions.
 
 This module can also be installed with drush make in your install profile.
 
+### External queue
+If external queue is selected from settings, `os2web_esdh_provider.queue.php`, must be
+executed using the servers crontab. This can be done using `drush php-script`, see
+<http://drush.ws/#php-script>
+
 Licence and copyright
 ---------------------
 OS2Web is Copyright (c) 2012 Syddjurs Kommune, Denmark

os2web_esdh_provider.admin.inc

@@ -38,19 +38,81 @@ function _os2web_esdh_provider_status() {
     $watchdog_head = array(t('Warning messages in descending order'), t('Date'));
     foreach ($watchdog_messages as $row) {
       $watchdog_rows[] = array(
-        t($row->message, unserialize($row->variables)),
+        format_string($row->message, unserialize($row->variables)),
         format_date($row->timestamp),
       );
     }
     $watchdog_table = theme('table', array('header' => $watchdog_head, 'rows' => $watchdog_rows));
   }
 
+  $queue_status = '';
+
+  if (variable_get('os2web_esdh_provider_queue_enabled', FALSE)) {
+    // Status for external meeting importer.
+    $queue_markup = '<h2>Meeting import: currently not running</h2>';
+
+    if (!lock_acquire('os2web_esdh_provider_queue')) {
+      $queue_markup = '<h2>Meeting import: running!</h2>';
+    }
+
+    $queue_markup .= drupal_render(drupal_get_form('os2web_esdh_provider_queue_form'));
+    $queue_markup .= '<h3>In queue</h3>';
+
+    $queue_header = array(t('Publication id:'), t('uri'), t('Force'));
+    $queue_rows = array();
+
+    $queue = DrupalQueue::get('acadre_mm_import');
+    while($item = $queue->claimItem()) {
+      if (isset($item->data['meeting'])) {
+        $queue_rows[] = array(
+          $item->data['meeting']['id'],
+          $item->data['meeting']['uri'],
+          $item->data['force'],
+        );
+      }
+    }
+
+    $queue_status .= '<br /><hr /><br />' . $queue_markup . theme('table', array(
+        'header' => $queue_header,
+        'rows' => $queue_rows,
+        'empty' => t('Queue is empty'),
+      ));
+
+    $queue_status .= '<br /><hr /><br />';
+  }
+
   $import_form = drupal_get_form('os2web_esdh_provider_import_form');
   return '<h2>' . t('ESDH Provider API plugin status') . '</h2>' .
       theme('table', array('header' => $head, 'rows' => $rows)) .
-      '<h2>' . t('Import of MM Based ESDH documents. (Last import was run at %time)', array('%time' => format_date(variable_get('os2web_esdh_provider_last_import', 0)))) . '</h2>' .
-      drupal_render($import_form) .
-      $watchdog_table;
+      '<h3>' . t('Last agenda import was successfully run at %time', array('%time' => format_date(variable_get('os2web_esdh_provider_last_import', 0)))) . '</h3>' .
+      '<h3>' . t('Last import job was run at %time', array('%time' => format_date(variable_get('os2web_esdh_provider_last_mm_import', 0)))) . '</h3>' .
+      drupal_render($import_form) . $queue_status . $watchdog_table;
+}
+
+/**
+ * Queue form.
+ */
+function os2web_esdh_provider_queue_form($form_state) {
+  $form = array();
+  $form['delete'] = array(
+    '#type' => 'submit',
+    '#value' => t('Delete queue'),
+  );
+
+  return $form;
+}
+
+/**
+ * Submit handler for queue from.
+ */
+function os2web_esdh_provider_queue_form_submit(&$form, &$form_state) {
+  // Empty queue.
+  $queue = DrupalQueue::get('acadre_mm_import');
+  while($item = $queue->claimItem()) {
+    $queue->deleteItem($item);
+  }
+
+  drupal_set_message(t('Deleted queue'));
 }
 
 /**
@@ -106,7 +168,6 @@ function os2web_esdh_provider_import_form_submit($form, &$form_state) {
       'force' => $form_state['values']['force'],
     ),
   ));
-  return;
 }
 
 /**
@@ -148,6 +209,7 @@ function _os2web_esdh_provider_status_confirm_submit($form, &$form_state) {
     drupal_set_message(t('ERROR: No valid MM importer plugin active!'), 'error');
     return;
   }
+
   $force = $form_state['values']['_meeting_force'] == 1;
   $focus = array_map('trim', explode(',', $form_state['values']['_meeting_id']));
   if (count($focus) === 1 && $focus[0] === '') {
@@ -160,20 +222,39 @@ function _os2web_esdh_provider_status_confirm_submit($form, &$form_state) {
     'file' => drupal_get_path('module', 'os2web_esdh_provider') . '/os2web_esdh_provider.mmapi.inc',
     'init_message' => t('Scanner filer'),
   );
+
   $meetings = os2web_esdh_provider_invoke('mm', 'get_import_list');
-  foreach ($meetings as $meeting) {
-    if (empty($focus) || in_array($meeting['id'], $focus)) {
-      $batch['operations'][] = array(
-        '_os2web_esdh_provider_import_meeting',
-        array($meeting, $force));
+
+  if (variable_get('os2web_esdh_provider_queue_enabled', FALSE)) {
+    $queue = DrupalQueue::get('acadre_mm_import');
+
+    foreach ($meetings as $meeting) {
+      if (empty($focus) || in_array($meeting['id'], $focus)) {
+        $queue->createItem(array('meeting' => $meeting, 'force' => $force));
+      }
     }
+    $queue->createItem(array('post_import_process' => TRUE));
+
+    drupal_goto('admin/config/os2web/esdh_provider');
   }
-  if (os2web_esdh_provider_supports('mm', 'post_import_process')) {
-    $batch['operations'][] = array(
-      'os2web_esdh_provider_invoke',
-      array('mm', 'post_import_process'));
+  // Use normal batch processing if external queue is disabled.
+  else {
+    foreach ($meetings as $meeting) {
+      if (empty($focus) || in_array($meeting['id'], $focus)) {
+        $batch['operations'][] = array(
+          '_os2web_esdh_provider_import_meeting',
+          array($meeting, $force));
+      }
+    }
+    if (os2web_esdh_provider_supports('mm', 'post_import_process')) {
+      $batch['operations'][] = array(
+        'os2web_esdh_provider_invoke',
+        array('mm', 'post_import_process'));
+    }
+
+    batch_set($batch);
+    // This isn't entirely neccesary, but needed to make the redirect work.
+    variable_set('os2web_esdh_provider_last_mm_import', $current_timestamp);
+    batch_process('admin/config/os2web/esdh_provider');
   }
-  batch_set($batch);
-  // This isn't entirely neccesary, but needed to make the redirect work.
-  batch_process('admin/config/os2web/esdh_provider');
 }

os2web_esdh_provider.module

@@ -1,5 +1,4 @@
 <?php
-
 /**
  * @file
  * Implements pluginable interface for accessing ESDH systems.
@@ -49,20 +48,36 @@ function os2web_esdh_provider_menu() {
     'access callback' => TRUE,
     'file' => 'os2web_esdh_provider.cpapi.inc',
   );
+
   return $items;
 }
 
 /**
  * Implements hook_FORM_alter().
  */
 function os2web_esdh_provider_form_os2web_settings_settings_form_alter(&$form, &$form_state) {
+  $form['os2web_esdh_provider'] = array(
+    '#type' => 'fieldset',
+    '#title' => 'Import',
+    '#description' => 'Generelle indstillinger vedr. import',
+    '#collapsible' => TRUE,
+    '#collapsed' => FALSE,
+  );
+
   // MM configuration.
-  $form['meetings']['os2web_meetings_import_service_ip'] = array(
+  $form['os2web_esdh_provider']['os2web_meetings_import_service_ip'] = array(
     '#type' => 'textfield',
     '#title' => 'IP addresser der kan udløse import',
     '#description' => 'Komma separeret liste af ip-addresser der kan tilgå <em>admin/os2web/import/<ID></em>.',
     '#default_value' => variable_get('os2web_meetings_import_service_ip', ip_address()),
   );
+
+  // Enable external queue import.
+  $form['os2web_esdh_provider']['os2web_esdh_provider_queue_enabled'] = array(
+    '#type' => 'checkbox',
+    '#title' => t('Enable external queue import'),
+    '#default_value' => variable_get('os2web_esdh_provider_queue_enabled', FALSE),
+  );
 }
 
 /**
@@ -144,7 +159,7 @@ function os2web_esdh_provider_node_delete($node) {
         node_delete($add_node->nid);
       }
     }
-    //Delete bullets from meeting.
+    // Delete bullets from meeting.
     $items = array_shift($node->field_os2web_meetings_bullets);
     if (is_array($items)) {
       foreach ($items as $nids) {
@@ -169,8 +184,13 @@ function os2web_esdh_provider_node_delete($node) {
  * Will run import on every cron.
  */
 function os2web_esdh_provider_cron() {
+  if (variable_get('os2web_esdh_provider_queue_enabled', FALSE)) {
+    return;
+  }
+
   if (os2web_esdh_provider_has_api('mm')) {
     $meetings = os2web_esdh_provider_invoke('mm', 'get_import_list');
+
     $queue = DrupalQueue::get('acadre_mm_import');
     if ($queue->numberOfItems() == 0) {
 
@@ -189,10 +209,40 @@ function os2web_esdh_provider_cron() {
   }
 }
 
+/**
+ * Worker function that queues meetings for external import.
+ */
+function os2web_esdh_provider_queue_meetings() {
+  if (os2web_esdh_provider_has_api('mm')) {
+    $meetings = os2web_esdh_provider_invoke('mm', 'get_import_list');
+
+    $last_mm_import_time = variable_get('os2web_esdh_provider_last_mm_import', FALSE);
+
+    $queue = DrupalQueue::get('acadre_mm_import');
+
+    foreach ($meetings as $meeting) {
+      // Only import files newer than last cron job.
+      if (!$last_mm_import_time || filemtime($meeting['uri']) > $last_mm_import_time) {
+        $queue->createItem(array('meeting' => $meeting, 'force' => FALSE));
+      }
+    }
+    $queue->createItem(array('post_import_process' => TRUE));
+    variable_set('os2web_esdh_provider_last_mm_import', $current_timestamp);
+
+    variable_set('os2web_esdh_provider_last_mm_import', time());
+  }
+}
+
 /**
  * Implements hook_cron_queue_info().
  */
 function os2web_esdh_provider_cron_queue_info() {
+  // If the external queue handler is enabled, don't let
+  // drupals cron impor the meetings.
+  if (variable_get('os2web_esdh_provider_queue_enabled', FALSE)) {
+    return array();
+  }
+
   $queues['acadre_mm_import'] = array(
     'worker callback' => '_os2web_esdh_provider_cron_queue_worker',
     'time' => 30,
@@ -541,7 +591,6 @@ function o2web_esdh_provider_help($path, $arg) {
  * Implements hook_os2web_help().
  */
 function o2web_esdh_provider_os2web_help($sections) {
-
   // Module specific.
   $sections['list_of_content'] .= t('<a href="#os2web_esdh_provider">OS2web ESDH Provider</a><br />');
   $sections['o2web_esdh_provider'] = t('<h2 id="os2web_esdh_provider">The ESDH Provider:</h2>');

os2web_esdh_provider.queue.php

@@ -0,0 +1,25 @@
+<?php
+/**
+ * @file
+ * os2web_esdh_provider.queue.php
+ *
+ * Let the meetings importer run via CLI instead of HTTP
+ *
+ * Run with Drush php-script
+ * "drush scr os2web_esdh_provider.queue.php"
+ */
+
+require_once 'os2web_esdh_provider.mmapi.inc';
+
+os2web_esdh_provider_queue_meetings();
+
+if (lock_acquire('os2web_esdh_provider_queue', 10000)) {
+  $queue = DrupalQueue::get('acadre_mm_import');
+
+  while($item = $queue->claimItem()) {
+    _os2web_esdh_provider_cron_queue_worker($item->data);
+    $queue->deleteItem($item);
+  }
+}
+
+lock_release('os2web_esdh_provider_queue');