test fixes

Author: soodoku

.github/workflows/ci.yml

@@ -16,28 +16,32 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.11', '3.12']
-    
+
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Install uv
         uses: astral-sh/setup-uv@v3
         with:
           version: "latest"
-      
+
       - name: Install dependencies
         run: |
           uv pip install --system -e ".[test]"
-      
+
+      - name: Install Playwright browsers
+        run: |
+          playwright install chromium
+
       - name: Run linting with ruff
         run: |
           ruff check piedomains/
-      
+
       - name: Run tests
         run: |
-          pytest tests/ -v -m "not ml and not performance" --cov=piedomains --cov-report=term-missing
+          pytest tests/ -v -m "not ml and not performance" --cov=piedomains --cov-report=term-missing

piedomains/api.py

@@ -10,7 +10,7 @@
 
 import os
 import re
-from datetime import datetime
+from datetime import UTC, datetime
 
 # LLM imports happen lazily when needed
 from .piedomains_logging import get_logger
@@ -660,7 +660,7 @@ def classify_from_collection(
                 os.makedirs(os.path.dirname(output_file), exist_ok=True)
 
                 output_data = {
-                    "inference_timestamp": datetime.utcnow().isoformat() + "Z",
+                    "inference_timestamp": datetime.now(UTC).isoformat(),
                     "model_used": "combined/text_image_ml",
                     "total_domains": len(combined_results),
                     "successful": len(

piedomains/content_processor.py

@@ -107,8 +107,6 @@ def extract_all_content(
                         fetch_results.append(result)
                     except Exception as e:
                         # Create a failed FetchResult for network errors
-                        from ..fetchers import FetchResult
-
                         failed_result = FetchResult(
                             url=url, success=False, error=f"Network error: {str(e)}"
                         )
@@ -334,6 +332,6 @@ def _parse_domain_name(self, url_or_domain: str) -> str:
             str: Clean domain name
         """
         # Import here to avoid circular imports
-        from ..piedomain import Piedomain
+        from .piedomain import Piedomain
 
         return Piedomain.parse_url_to_domain(url_or_domain)

piedomains/data_collector.py

@@ -8,7 +8,7 @@
 
 import json
 import uuid
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 
 from .config import get_config
@@ -96,7 +96,7 @@ def collect(
             raise ValueError("domains list cannot be empty")
 
         collection_id = collection_id or str(uuid.uuid4())
-        collection_start = datetime.utcnow()
+        collection_start = datetime.now(UTC)
 
         logger.info(f"Starting data collection for {len(domains)} domains")
 
@@ -147,7 +147,7 @@ def _collect_single_domain(self, domain: str, use_cache: bool) -> dict:
             Dictionary with domain collection results
         """
         domain_name = self._parse_domain_name(domain)
-        collection_time = datetime.utcnow()
+        collection_time = datetime.now(UTC)
 
         # Define file paths
         html_file = self.html_dir / f"{domain_name}.html"
@@ -246,7 +246,7 @@ def collect_batch(
             raise ValueError("domains list cannot be empty")
 
         collection_id = collection_id or str(uuid.uuid4())
-        collection_start = datetime.utcnow()
+        collection_start = datetime.now(UTC)
 
         logger.info(f"Starting batch data collection for {len(domains)} domains")
 
@@ -273,7 +273,7 @@ def collect_batch(
                             "domain": domain_name,
                             "text_path": None,
                             "image_path": None,
-                            "date_time_collected": datetime.utcnow().isoformat() + "Z",
+                            "date_time_collected": datetime.now(UTC).isoformat(),
                             "fetch_success": False,
                             "cached": False,
                             "error": f"Batch processing failed: {e}",
@@ -341,7 +341,7 @@ def _collect_batch_parallel(
                         "domain": domain_name,
                         "text_path": str(html_file.relative_to(self.cache_dir)),
                         "image_path": str(image_file.relative_to(self.cache_dir)),
-                        "date_time_collected": datetime.utcnow().isoformat() + "Z",
+                        "date_time_collected": datetime.now(UTC).isoformat(),
                         "fetch_success": True,
                         "cached": True,
                         "error": None,
@@ -363,7 +363,7 @@ def _collect_batch_parallel(
                 fetched_results = []
                 for result in fetch_results:
                     domain_name = self._parse_domain_name(result.url)
-                    collection_time = datetime.utcnow()
+                    collection_time = datetime.now(UTC)
 
                     if result.success:
                         # Save HTML content

piedomains/image.py

@@ -4,6 +4,7 @@
 """
 
 import os
+from datetime import UTC
 from pathlib import Path
 
 import numpy as np
@@ -239,7 +240,7 @@ def classify_from_paths(
 
             # Add metadata
             output_data = {
-                "inference_timestamp": datetime.utcnow().isoformat() + "Z",
+                "inference_timestamp": datetime.now(UTC).isoformat(),
                 "model_used": "image/shallalist_ml",
                 "total_domains": len(data_paths),
                 "successful": len([r for r in results if r["category"] is not None]),

piedomains/llm_classifier.py

@@ -5,7 +5,7 @@
 import base64
 import json
 import time
-from datetime import datetime
+from datetime import UTC, datetime
 from io import BytesIO
 from pathlib import Path
 from typing import Any
@@ -557,7 +557,7 @@ def classify_from_paths(
 
             # Add metadata
             output_data = {
-                "inference_timestamp": datetime.utcnow().isoformat() + "Z",
+                "inference_timestamp": datetime.now(UTC).isoformat(),
                 "model_used": f"{mode}/llm_{self.config.provider}_{self.config.model}",
                 "total_domains": len(data_paths),
                 "successful": len([r for r in results if r["category"] is not None]),

piedomains/text.py

@@ -4,6 +4,7 @@
 """
 
 import os
+from datetime import UTC
 from pathlib import Path
 
 import numpy as np
@@ -304,7 +305,7 @@ def classify_from_paths(
 
             # Add metadata
             output_data = {
-                "inference_timestamp": datetime.utcnow().isoformat() + "Z",
+                "inference_timestamp": datetime.now(UTC).isoformat(),
                 "model_used": "text/shallalist_ml",
                 "total_domains": len(data_paths),
                 "successful": len([r for r in results if r["category"] is not None]),