Merge pull request #107 from nuest/fix-ci

Fix CI

Author: nuest

.github/workflows/django.yml

@@ -15,8 +15,8 @@ jobs:
       max-parallel: 4
       matrix:
         python-version: [
-          3.8,
-          #3.9
+          3.11,
+          #3.13
         ]
 
     services:
@@ -75,7 +75,7 @@ jobs:
 
     - name: Upload screenshots
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: screenshots for Python ${{ matrix.python-version }}
         path: |

README.md

@@ -109,11 +109,15 @@ source .venv/bin/activate
 # Confirm Python path
 which python
 
-# Install required dependencies
+# Instal GDAL and the Python GDAL bindings, see Dockerfile for example on Ubuntu
+gdalinfo --version
+
+# Install non-GDAL Python dependencies
 pip install -r requirements.txt
 
-# Create a local database container (once only)
-docker run --name optimapDB -p 5432:5432 -e POSTGRES_USER=optimap -e POSTGRES_PASSWORD=optimap -e POSTGRES_DB=optimap -d postgis/postgis:15-3.4
+# create local DB container (once)
+# docker run --name optimapDB -p 5432:5432 -e POSTGRES_USER=optimap -e POSTGRES_PASSWORD=optimap -e POSTGRES_DB=optimap -d postgis/postgis:14-3.3
+# get a clean one later: docker rm -f optimapDB
 
 # Start the database container
 docker start optimapDB
@@ -190,22 +194,6 @@ Configuration for debugging with VS Code:
       "justMyCode": true
     }
   ]
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Python: Django Run",
-      "type": "python",
-      "request": "launch",
-      "program": "${workspaceFolder}/manage.py",
-      "args": ["runserver"],
-      "env": {
-        "OPTIMAP_DEBUG": "True",
-        "OPTIMAP_CACHE": "dummy"
-      },
-      "django": true,
-      "justMyCode": true
-    }
-  ]
 }
 ```
 
@@ -306,17 +294,6 @@ A configuration to debug the test code and also print deprecation warnings:
   },
   "django": true,
   "justMyCode": true
-  "name": "Python: Django Test",
-  "type": "python",
-  "request": "launch",
-  "pythonArgs": ["-Wa"],
-  "program": "${workspaceFolder}/manage.py",
-  "args": ["test", "tests"],
-  "env": {
-    "OPTIMAP_DEBUG": "True"
-  },
-  "django": true,
-  "justMyCode": true
 }
 ```
 

optimap/urls.py

@@ -16,8 +16,8 @@
 from django.contrib import admin
 from django.urls import path, re_path, include
 from django.contrib.sitemaps import views as sitemaps_views
-from django.http import HttpResponse
 from publications.sitemaps import PublicationsSitemap, StaticViewSitemap
+from publications.views import RobotsView
 
 sitemaps = {
     "static": StaticViewSitemap,
@@ -39,7 +39,7 @@
         {"sitemaps": sitemaps},
         name="django.contrib.sitemaps.views.sitemap",
     ),
-    re_path(r'^robots.txt', lambda request: HttpResponse("User-Agent: *\nDisallow:\nSitemap: %s://%s/sitemap.xml" % (request.scheme, request.site.domain), content_type="text/plain"), name="robots_file"),
+    re_path(r'^robots.txt', RobotsView.as_view(), name="robots_file"),
     ]
 
 # https://stackoverflow.com/a/18272203/261210

publications/migrations/0006_merge_20250317_1458.py

@@ -0,0 +1,14 @@
+# Generated by Django 5.1.7 on 2025-03-17 14:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('publications', '0004_blockeddomain_blockedemail'),
+        ('publications', '0005_sentemaillog_sent_by'),
+    ]
+
+    operations = [
+    ]

publications/migrations/0007_blockeddomain_blocked_by_blockedemail_blocked_by_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.1.7 on 2025-03-17 14:59
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('publications', '0006_merge_20250317_1458'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blockeddomain',
+            name='blocked_by',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='blocked_domains', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='blockedemail',
+            name='blocked_by',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='blocked_emails', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.CreateModel(
+            name='EmailLog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('recipient_email', models.EmailField(max_length=254)),
+                ('subject', models.CharField(max_length=255)),
+                ('sent_at', models.DateTimeField(auto_now_add=True)),
+                ('email_content', models.TextField(blank=True, null=True)),
+                ('trigger_source', models.CharField(choices=[('admin', 'Admin Panel'), ('scheduled', 'Scheduled Task'), ('manual', 'Manually Triggered')], default='manual', max_length=50)),
+                ('status', models.CharField(choices=[('d', 'Draft'), ('p', 'Published'), ('t', 'Testing'), ('w', 'Withdrawn'), ('h', 'Harvested')], default='success', max_length=10)),
+                ('error_message', models.TextField(blank=True, null=True)),
+                ('sent_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='UserProfile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('notify_new_manuscripts', models.BooleanField(default=False)),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='SentEmailLog',
+        ),
+    ]

publications/migrations/0008_alter_publication_doi.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.7 on 2025-03-17 15:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('publications', '0007_blockeddomain_blocked_by_blockedemail_blocked_by_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='publication',
+            name='doi',
+            field=models.CharField(blank=True, max_length=1024, unique=True),
+        ),
+    ]

publications/migrations/0009_alter_publication_doi.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.7 on 2025-03-17 15:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('publications', '0008_alter_publication_doi'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='publication',
+            name='doi',
+            field=models.CharField(blank=True, max_length=1024, null=True, unique=True),
+        ),
+    ]

publications/migrations/0010_alter_publication_title.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.7 on 2025-03-17 23:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('publications', '0009_alter_publication_doi'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='publication',
+            name='title',
+            field=models.TextField(),
+        ),
+    ]

publications/models.py

@@ -16,7 +16,7 @@
 
 class Publication(models.Model):
     # required fields      
-    doi = models.CharField(max_length=1024, unique=True)
+    title = models.TextField()
     status = models.CharField(max_length=1, choices=STATUS_CHOICES, default="d")
     created_by = CurrentUserField( # see useful hint at https://github.com/zsoldosp/django-currentuser/issues/69
         verbose_name=("Created by"),
@@ -33,10 +33,10 @@ class Publication(models.Model):
     )
 
     # optional fields
+    doi = models.CharField(max_length=1024, unique=True, blank=True, null=True)
     source = models.CharField(max_length=4096, null=True, blank=True) # journal, conference, preprint repo, ..
     provenance = models.TextField(null=True, blank=True)
-    publicationDate = models.DateField(null=True,blank=True)
-    title = models.TextField(null=True, blank=True)
+    publicationDate = models.DateField(null=True, blank=True)
     abstract = models.TextField(null=True, blank=True)
     url = models.URLField(max_length=1024, null=True, blank=True)
     geometry = models.GeometryCollectionField(verbose_name='Publication geometry/ies', srid = 4326, null=True, blank=True)# https://docs.openalex.org/api-entities/sources

publications/tasks.py

@@ -28,7 +28,7 @@ def extract_geometry_from_html(content):
                 geom = json.loads(data)
 
                 geom_data = geom["features"][0]["geometry"]
-                # preparing geometry data in accordance to geosAPI fields
+                # preparing geometry data in accordance to geos API fields
                 type_geom= {'type': 'GeometryCollection'}
                 geom_content = {"geometries" : [geom_data]}
                 type_geom.update(geom_content)
@@ -37,10 +37,10 @@ def extract_geometry_from_html(content):
                     geom_object = GEOSGeometry(geom_data_string) # GeometryCollection object
                     logging.debug('Found geometry: %s', geom_object)
                     return geom_object
-                except :
-                    print("Invalid Geometry")
+                except Exception as e:
+                    logger.error("Cannot create geometry from string '%s': %s", geom_data_string, e)
             except ValueError as e:
-                print("Not a valid GeoJSON")
+                logger.error("Error loading JSON from %s: %s", tag.get("name"), e)
 
 def extract_timeperiod_from_html(content):
     period = [None, None]
@@ -61,13 +61,20 @@ def parse_oai_xml_and_save_publications(content):
     for i in range(articles_count_in_journal):
         identifier = collection.getElementsByTagName("dc:identifier")
         identifier_value = identifier[i].firstChild.nodeValue
+        logger.debug("Retrieving %s", identifier_value)
+        
         if identifier_value.startswith('http'):
 
-            with requests.get(identifier_value) as response:
-                soup = BeautifulSoup(response.content, 'html.parser')
+            try:
+                with requests.get(identifier_value) as response:
+                    soup = BeautifulSoup(response.content, 'html.parser')
 
-                geom_object = extract_geometry_from_html(soup)
-                period_start, period_end = extract_timeperiod_from_html(soup)
+                    geom_object = extract_geometry_from_html(soup)
+                    period_start, period_end = extract_timeperiod_from_html(soup)
+            except Exception as e:
+                logger.error("Error retrieving and extracting geometadata from URL %s: %s", identifier_value, e)
+                logger.error("Continueing with the next article...")
+                continue
 
         else:
             geom_object = None
@@ -100,20 +107,20 @@ def parse_oai_xml_and_save_publications(content):
             abstract = abstract_text,
             publicationDate = date_value,
             url = identifier_value,
-            journal = journal_value,
+            source = journal_value,
             geometry = geom_object,
             timeperiod_startdate = period_start,
             timeperiod_enddate = period_end)
         publication.save()
-        logger.info('Saved new publication for %s: %s', identifier_value, publication)
+        logger.info('Saved new publication for %s: %s', identifier_value, publication.get_absolute_url())
 
 def harvest_oai_endpoint(url):
     try:
         with requests.Session() as s:
             response = s.get(url)
             parse_oai_xml_and_save_publications(response.content)
     except requests.exceptions.RequestException as e:
-        print ("The requested URL is invalid or has bad connection.Please change the URL")
+        logger.error("The requested URL is invalid or has bad connection. Please check the URL: %s", url)
 
 def send_monthly_email(trigger_source='manual', sent_by=None):
     recipients = User.objects.filter(userprofile__notify_new_manuscripts=True).values_list('email', flat=True)