Merge pull request #104 from pitangainnovare/impl/optimize-index

Impl/optimize index

Author: pitangainnovare

VERSION

@@ -1 +1 @@
-1.12.1
+1.13.0

metrics/tests/test_models.py article/management/__init__.pymetrics/tests/test_models.py renamed to article/management/__init__.py

@@ -0,0 +1,80 @@
+from django.core.management.base import BaseCommand
+
+from article.tasks import task_load_article_from_opac, task_load_article_from_article_meta
+
+
+class Command(BaseCommand):
+    help = 'Generate task requests for loading article data from Article Meta for each year from 1900 to 2025'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--start-year',
+            type=int,
+            default=1990,
+            help='Start year (default: 1990)'
+        )
+        parser.add_argument(
+            '--end-year',
+            type=int,
+            default=2025,
+            help='End year (default: 2025)'
+        )
+        parser.add_argument(
+            '--collection',
+            type=str,
+            default='scl',
+            help='Collection code (default: scl)'
+        )
+        parser.add_argument(
+            '--task',
+            choices=['load_article_from_opac', 'load_article_from_article_meta'],
+            default='load_article_from_opac',
+            help='Task to execute (default: load_article_from_opac)',
+        )
+
+    def handle(self, *args, **options):
+        start_year = options['start_year']
+        end_year = options['end_year']
+        collection = options['collection']
+        
+        self.stdout.write(
+            self.style.SUCCESS(
+                f'Generating task requests from {start_year} to {end_year} for collection: {collection}'
+            )
+        )
+        
+        total_tasks = 0
+        
+        for year in range(start_year, end_year + 1):
+            from_date = f'{year}-01-01'
+            until_date = f'{year}-12-31'
+            
+            self.stdout.write(f'Queuing task for year {year}...')
+            
+            # Queue the task for each year
+            if options['task'] == 'load_article_from_article_meta':
+                task_result = task_load_article_from_article_meta.delay(
+                    from_date=from_date,
+                    until_date=until_date,
+                    collection=collection
+                )
+            else:
+                task_result = task_load_article_from_opac.delay(
+                    from_date=from_date,
+                    until_date=until_date,
+                    collection=collection
+                )
+            
+            total_tasks += 1
+            
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f'✓ Task queued for year {year}: {from_date} to {until_date} (Task ID: {task_result.id})'
+                )
+            )
+        
+        self.stdout.write(
+            self.style.SUCCESS(
+                f'\nCompleted! {total_tasks} tasks have been queued successfully.'
+            )
+        )

article/management/commands/__init__.py

@@ -114,7 +114,6 @@ def metadata(cls, collection=None):
 
         for a in qs.iterator():
             yield {
-                'id': a.id,
                 'collection': a.collection.acron3,
                 'default_lang': a.default_lang,
                 'files': a.files,

article/management/commands/load_articles_by_year.py

@@ -60,7 +60,7 @@ def task_load_article_from_article_meta(self, from_date=None, until_date=None, d
             try:
                 article, created = models.Article.objects.get_or_create(collection=col_obj, scielo_issn=jou.scielo_issn, pid_v2=obj.get('code'))
                 if created or force_update:
-                    article.files = obj.get('files') or {}
+                    article.files = obj.get('pdfs') or {}
                     article.processing_date = obj.get('processing_date') or ''
                     article.publication_date = obj.get('publication_date') or ''
                     article.publication_year = obj.get('publication_year') or ''
@@ -121,8 +121,8 @@ def task_load_article_from_opac(self, collection='scl', from_date=None, until_da
                     article.pid_v3 = doc.get('pid_v3') or ''
                     if not created:
                         article.pid_v2 = doc.get('pid_v2') or ''
-                        article.publication_date = doc.get('publication_date') or ''
-                        article.default_lang = doc.get('default_language') or ''
+                        article.publication_date = doc.get('publication_date') or article.publication_date or ''
+                        article.default_lang = doc.get('default_language') or article.default_lang or ''
 
                         try:
                             article.publication_year = article.publication_date[:4]

article/models.py

@@ -22,7 +22,7 @@ class ArticleSnippetViewSet(SnippetViewSet):
         "pid_v3",
         "pid_generic",
         "files",
-        "publication_date",
+        "publication_year",
     )
     list_filter = (
         "collection",

article/tasks.py

@@ -4,4 +4,4 @@ set -o errexit
 set -o nounset
 
 
-watchgod celery.__main__.main --args -A config.celery_app worker -l INFO --concurrency=4
+watchgod celery.__main__.main --args -A config.celery_app worker -l INFO --concurrency=1

article/wagtail_hooks.py

@@ -5,4 +5,4 @@ set -o pipefail
 set -o nounset
 
 
-exec celery -A config.celery_app worker -l INFO --concurrency=4
+exec celery -A config.celery_app worker -l INFO --concurrency=1

compose/local/django/celery/worker/start

@@ -407,7 +407,7 @@
 # Elasticsearch
 # ------------------------------------------------------------------------------
 ES_URL = env("ES_URL", default="http://192.168.0.33:9200/")
-ES_INDEX_NAME = env("ES_INDEX_NAME", default="usage-daily")
+ES_INDEX_NAME = env("ES_INDEX_NAME", default="usage")
 ES_API_KEY = env("ES_API_KEY", default="")
 ES_BASIC_AUTH = env("ES_BASIC_AUTH", default=("elastic", "iHktg66E"))
 ES_VERIFY_CERTS = env.bool("ES_VERIFY_CERTS", default=False)