Fix CI/CD pipeline issues and code quality

- Updated GitHub Actions to use latest action versions (@v4)
- Standardized test commands across all platforms to exclude ML tests
- Fixed flake8 linting issues: removed unused imports, cleaned whitespace
- Streamlined dependency installation in workflows
- All non-ML tests now pass consistently (79 passed, 3 deselected)

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: soodoku

piedomains/tests/__init__.py

@@ -1,6 +1,5 @@
 import sys
 from contextlib import contextmanager
-import pandas
 
 try:
     from StringIO import StringIO

piedomains/tests/images/khanacademy.org.png

@@ -19,7 +19,7 @@ def test_validate_domain_name_valid(self):
             "a.b",
             "very-long-subdomain.example-domain.com"
         ]
-        
+
         for domain in valid_domains:
             with self.subTest(domain=domain):
                 self.assertTrue(Piedomain.validate_domain_name(domain))
@@ -38,7 +38,7 @@ def test_validate_domain_name_invalid(self):
             "[email protected]",
             "just-a-string-without-dot"
         ]
-        
+
         for domain in invalid_domains:
             with self.subTest(domain=domain):
                 self.assertFalse(Piedomain.validate_domain_name(domain))
@@ -50,7 +50,7 @@ def test_validate_domain_name_with_protocol(self):
             "https://example.org",
             "https://sub.example.com/path"
         ]
-        
+
         for domain in domains_with_protocol:
             with self.subTest(domain=domain):
                 self.assertTrue(Piedomain.validate_domain_name(domain))
@@ -64,13 +64,13 @@ def test_validate_domains_list(self):
             "",
             "twitter.com"
         ]
-        
+
         valid, invalid = Piedomain.validate_domains(mixed_domains)
-        
+
         self.assertEqual(len(valid), 3)
         self.assertEqual(len(invalid), 2)
         self.assertIn("google.com", valid)
-        self.assertIn("facebook.com", valid) 
+        self.assertIn("facebook.com", valid)
         self.assertIn("twitter.com", valid)
         self.assertIn("invalid..domain", invalid)
         self.assertIn("", invalid)
@@ -82,9 +82,9 @@ def test_validate_domains_normalization(self):
             "http://test.org/path",
             "example.net/"
         ]
-        
+
         valid, invalid = Piedomain.validate_domains(domains_to_normalize)
-        
+
         self.assertEqual(len(valid), 3)
         self.assertEqual(len(invalid), 0)
         self.assertIn("example.com", valid)
@@ -93,4 +93,5 @@ def test_validate_domains_normalization(self):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()
+

piedomains/tests/images/yahoo.com.png

@@ -21,9 +21,9 @@ def test_text_from_html_basic(self):
         </body>
         </html>
         """
-        
+
         result = Piedomain.text_from_html(html_content)
-        
+
         self.assertIsInstance(result, str)
         self.assertIn("welcome", result.lower())
         self.assertIn("test", result.lower())
@@ -44,9 +44,9 @@ def test_text_from_html_with_scripts_and_styles(self):
         </body>
         </html>
         """
-        
+
         result = Piedomain.text_from_html(html_content)
-        
+
         # Should extract text but not script/style content
         self.assertIn("main", result.lower())
         self.assertIn("content", result.lower())
@@ -56,7 +56,7 @@ def test_data_cleanup_removes_numbers(self):
         """Test that data cleanup removes numbers."""
         text_with_numbers = "test123 content456 more789 text"
         result = Piedomain.data_cleanup(text_with_numbers)
-        
+
         self.assertNotIn("123", result)
         self.assertNotIn("456", result)
         self.assertNotIn("789", result)
@@ -65,7 +65,7 @@ def test_data_cleanup_removes_punctuation(self):
         """Test that data cleanup removes punctuation."""
         text_with_punct = "hello, world! this is a test."
         result = Piedomain.data_cleanup(text_with_punct)
-        
+
         self.assertNotIn(",", result)
         self.assertNotIn("!", result)
         self.assertNotIn(".", result)
@@ -74,7 +74,7 @@ def test_data_cleanup_removes_stopwords(self):
         """Test that data cleanup removes English stopwords."""
         text_with_stopwords = "the quick brown fox and jumps in the lazy dog"
         result = Piedomain.data_cleanup(text_with_stopwords)
-        
+
         # Common stopwords should be removed
         self.assertNotIn(" the ", " " + result + " ")
         self.assertNotIn(" and ", " " + result + " ")
@@ -88,7 +88,7 @@ def test_data_cleanup_removes_short_words(self):
         """Test that data cleanup removes single character words."""
         text_with_short = "a big test i o u"
         result = Piedomain.data_cleanup(text_with_short)
-        
+
         # Single characters should be removed
         result_words = result.split()
         for word in result_words:
@@ -98,17 +98,17 @@ def test_data_cleanup_lowercase(self):
         """Test that data cleanup converts to lowercase."""
         text_mixed_case = "This IS Mixed CASE Text"
         result = Piedomain.data_cleanup(text_mixed_case)
-        
+
         self.assertEqual(result, result.lower())
 
     def test_data_cleanup_removes_duplicates(self):
         """Test that data cleanup removes duplicate words."""
         text_with_duplicates = "test test content content more test"
         result = Piedomain.data_cleanup(text_with_duplicates)
-        
+
         words = result.split()
         unique_words = set(words)
-        
+
         # Should have same number of unique words as total words
         self.assertEqual(len(words), len(unique_words))
 
@@ -117,7 +117,7 @@ def test_data_cleanup_filters_non_english(self):
         # This test may be limited by the NLTK words corpus availability
         text_mixed = "computer test français deutsche invalid"
         result = Piedomain.data_cleanup(text_mixed)
-        
+
         # Should contain recognizable English words that aren't stopwords
         self.assertIn("computer", result)
         self.assertIn("test", result)
@@ -134,4 +134,5 @@ def test_data_cleanup_only_numbers_and_punct(self):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()
+

piedomains/tests/test_004_domain_validation.py

@@ -29,26 +29,26 @@ def test_validate_input_empty_list_no_path(self):
         """Test validate_input with empty domain list and no path."""
         with self.assertRaises(Exception) as context:
             Piedomain.validate_input([], None, "html")
-        
+
         self.assertIn("Provide list of Domains", str(context.exception))
 
     def test_validate_input_nonexistent_path(self):
         """Test validate_input with nonexistent path."""
         nonexistent_path = "/path/that/does/not/exist"
-        
+
         with self.assertRaises(Exception) as context:
             Piedomain.validate_input([], nonexistent_path, "html")
-        
+
         self.assertIn("does not exist", str(context.exception))
 
     def test_validate_input_empty_directory(self):
         """Test validate_input with empty directory."""
         empty_dir = os.path.join(self.temp_dir, "empty")
         os.makedirs(empty_dir)
-        
+
         with self.assertRaises(Exception) as context:
             Piedomain.validate_input([], empty_dir, "html")
-        
+
         self.assertIn("is empty", str(context.exception))
 
     def test_validate_input_valid_offline_mode(self):
@@ -57,18 +57,18 @@ def test_validate_input_valid_offline_mode(self):
         test_file = os.path.join(self.html_dir, "test.html")
         with open(test_file, "w") as f:
             f.write("<html><body>test</body></html>")
-        
+
         result = Piedomain.validate_input([], self.html_dir, "html")
         self.assertTrue(result)  # Should return True for offline mode
 
     @patch('requests.get')
     def test_extract_htmls_network_error(self, mock_get):
         """Test HTML extraction with network errors."""
         mock_get.side_effect = Exception("Network error")
-        
+
         domains = ["example.com"]
         errors = Piedomain.extract_htmls(domains, False, self.html_dir)
-        
+
         self.assertIn("example.com", errors)
         self.assertIn("Network error", str(errors["example.com"]))
 
@@ -78,10 +78,10 @@ def test_extract_htmls_http_error(self, mock_get):
         mock_response = MagicMock()
         mock_response.raise_for_status.side_effect = Exception("404 Not Found")
         mock_get.return_value = mock_response
-        
+
         domains = ["nonexistent.com"]
         errors = Piedomain.extract_htmls(domains, False, self.html_dir)
-        
+
         self.assertIn("nonexistent.com", errors)
 
     @patch('piedomains.piedomain.Piedomain.get_driver')
@@ -90,9 +90,9 @@ def test_save_image_webdriver_error(self, mock_get_driver):
         mock_driver = MagicMock()
         mock_driver.get.side_effect = Exception("WebDriver error")
         mock_get_driver.return_value = mock_driver
-        
+
         success, error_msg = Piedomain.save_image("example.com", self.image_dir)
-        
+
         self.assertFalse(success)
         self.assertIn("WebDriver error", error_msg)
         mock_driver.quit.assert_called_once()
@@ -104,34 +104,34 @@ def test_save_image_driver_quit_error(self, mock_get_driver):
         mock_driver.get.side_effect = Exception("WebDriver error")
         mock_driver.quit.side_effect = Exception("Quit error")
         mock_get_driver.return_value = mock_driver
-        
+
         # Should handle quit error gracefully
         success, error_msg = Piedomain.save_image("example.com", self.image_dir)
-        
+
         self.assertFalse(success)
         self.assertIn("WebDriver error", error_msg)
 
     def test_extract_image_tensor_invalid_directory(self):
         """Test image tensor extraction with invalid directory."""
         nonexistent_dir = "/path/that/does/not/exist"
-        
+
         with self.assertRaises(FileNotFoundError):
             Piedomain.extract_image_tensor(True, ["example.com"], nonexistent_dir)
 
     def test_extract_html_text_invalid_directory(self):
         """Test HTML text extraction with invalid directory."""
         nonexistent_dir = "/path/that/does/not/exist"
-        
+
         with self.assertRaises(FileNotFoundError):
             Piedomain.extract_html_text(True, ["example.com"], nonexistent_dir)
 
     def test_text_from_html_malformed_html(self):
         """Test text extraction from malformed HTML."""
         malformed_html = "<html><body><p>Unclosed paragraph<div>Nested incorrectly</p></div></body></html>"
-        
+
         # Should handle malformed HTML gracefully
         result = Piedomain.text_from_html(malformed_html)
-        
+
         self.assertIsInstance(result, str)
         # BeautifulSoup should handle malformed HTML
 
@@ -146,7 +146,7 @@ def test_extract_images_permission_error(self):
         # Create a directory and remove write permissions
         restricted_dir = os.path.join(self.temp_dir, "restricted")
         os.makedirs(restricted_dir)
-        
+
         # This test may not work on all systems due to permission handling
         try:
             os.chmod(restricted_dir, 0o444)  # Read-only
@@ -160,13 +160,14 @@ def test_extract_images_permission_error(self):
     def test_validate_domains_with_none_values(self):
         """Test domain validation with None values in list."""
         domains_with_none = ["google.com", None, "facebook.com"]
-        
+
         valid, invalid = Piedomain.validate_domains(domains_with_none)
-        
+
         self.assertEqual(len(invalid), 1)
         self.assertIn(None, invalid)
         self.assertEqual(len(valid), 2)
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()
+