added optional delay to item indexing

Author: nwoodward

config/application.yml.sample

@@ -13,6 +13,7 @@ statistics_db:
     username: 'dspace_statistics'
     password: 'dspace_statistics'
 work_dir: '/tmp'
+delay: 0
 create_zip_archive: false
 log_path: 'logs'
 log_file: 'statistics-reports.log'

dspace_reports/item_indexer.py

@@ -1,6 +1,8 @@
 import math
 import sys
 
+from time import sleep
+
 from lib.oai import DSpaceOai
 from lib.database import Database
 from dspace_reports.indexer import Indexer
@@ -19,6 +21,9 @@ def __init__(self, config, logger):
         # Set time periods to only month and year as all can cause Solr to crash
         self.time_periods = ['month', 'year', 'all']
 
+        # Set crawl delay from config
+        self.delay = config['delay']
+
     def index(self):
         # Get list of identifiers from OAI-PMH feed
         records = self.oai.get_records()
@@ -190,6 +195,9 @@ def index_item_views(self, time_period='all'):
                     # Commit changes to database
                     db.commit()
 
+                    if self.delay:
+                        sleep(self.delay)
+
                     results_current_page += 1
 
     def index_item_downloads(self, time_period='all'):
@@ -298,4 +306,7 @@ def index_item_downloads(self, time_period='all'):
                     # Commit changes to database
                     db.commit()
 
+                    if self.delay:
+                        sleep(self.delay)
+
                     results_current_page += 1