Merge pull request #27 from DMontgomery40/refactor/modular-scanner

Refactor/modular scanner

Author: DMontgomery40

src/components/ProgressBar.jsx

@@ -12,14 +12,25 @@ const ProgressBar = ({ progress, onCancel, scanning }) => {
         />
       </div>
       <div className="text-sm text-gray-300 mt-2 text-center">
-        {progress.phase === 'fetching' && progress.total > 0
+        {progress.phase === 'fetching' && progress.total > 0 && progress.details?.successCount !== undefined
+          ? `Fetching files (${progress.current}/${progress.total}) • ✓${progress.details.successCount} ✗${progress.details.failureCount}`
+          : progress.phase === 'fetching' && progress.total > 0
           ? `Fetching files (${progress.current} of ${progress.total})`
+          : progress.phase === 'analyzing' && progress.total > 0 && progress.details?.successCount !== undefined
+          ? `Analyzing files (${progress.current}/${progress.total}) • ✓${progress.details.successCount} ✗${progress.details.failureCount}`
+          : progress.phase === 'completed' && progress.details?.summary
+          ? progress.details.summary
           : progress.phase === 'analyzing' && progress.details?.currentFile
           ? `Analyzing: ${progress.details.currentFile} (${progress.current} of ${progress.total})`
           : progress.phase === 'complete'
           ? 'Scan complete!'
           : `${progress.phase}: ${progress.current} of ${progress.total}`}
       </div>
+      {progress.phase === 'completed' && progress.details?.failureCount > 0 && (
+        <div className="text-xs text-yellow-400 mt-1 text-center">
+          ⚠️ {progress.details.failureCount} files couldn't be downloaded - scan may be incomplete
+        </div>
+      )}
       {onCancel && (
         <div className="text-center mt-2">
           <button

src/context/ScanContext.jsx

@@ -134,7 +134,7 @@ export const ScanProvider = ({ children }) => {
     });
 
     try {
-      const results = await scanRepositoryLocally(urlInput);
+      const results = await scanRepositoryLocally(urlInput, handleProgress);
       console.log('Scan results:', results);
 
       if (results.findings && results.summary) {
@@ -166,12 +166,19 @@ export const ScanProvider = ({ children }) => {
           }
         });
 
+        // Include scan statistics in success message
+        const scanStatsMsg = results.scanStats 
+          ? ` • Scanned ${results.scanStats.successCount}/${results.scanStats.totalFiles} files (${results.scanStats.completionRate}%) in ${results.scanStats.duration}s`
+          : '';
+        
+        const partialMsg = results.partial ? ' • ⚠️ Some files failed to download' : '';
+        
         setSuccessMessage(
           `Scan complete! Found ${results.summary.totalIssues} potential vulnerabilities ` +
           `(${results.summary.criticalIssues} critical, ` +
           `${results.summary.highIssues} high, ` +
           `${results.summary.mediumIssues} medium, ` +
-          `${results.summary.lowIssues} low)`
+          `${results.summary.lowIssues} low)${scanStatsMsg}${partialMsg}`
         );
 
         setUsedCache(results.fromCache || false);
@@ -203,11 +210,16 @@ export const ScanProvider = ({ children }) => {
     setError(null);
     setScanResults(null);
     setSuccessMessage('');
+    const startTime = Date.now();
     setProgress({
       phase: 'fetching',
       current: 0,
       total: 0,
-      details: { url }
+      details: { 
+        url,
+        status: 'Fetching webpage content...',
+        startTime
+      }
     });
     setFirmwareMessage('');
 
@@ -259,11 +271,53 @@ export const ScanProvider = ({ children }) => {
           }
         });
 
+        // Calculate scan duration
+        const endTime = Date.now();
+        const duration = Math.round((endTime - startTime) / 1000 * 100) / 100;
+        
+        // Count scanned items (HTML + scripts)
+        const scriptCount = data.scriptsScanned || 0;
+        const totalScanned = scriptCount + 1; // +1 for HTML page
+        
+        // Send completion progress
+        setProgress({
+          phase: 'completed',
+          current: totalScanned,
+          total: totalScanned,
+          details: {
+            duration: duration,
+            successCount: totalScanned,
+            failureCount: 0,
+            completionRate: 100,
+            totalAttempted: totalScanned,
+            summary: `Scanned ${totalScanned} items (HTML + ${scriptCount} scripts) in ${duration}s`
+          }
+        });
+
         setSuccessMessage(
-          `Website scan complete! Found ${summary.totalIssues || 0} potential vulnerabilities.`
+          `Website scan complete! Found ${summary.totalIssues || 0} potential vulnerabilities • ` +
+          `Scanned ${totalScanned} items in ${duration}s`
         );
       } else {
-        setSuccessMessage('Website scan completed, but no vulnerabilities reported.');
+        // Calculate scan duration even for no results
+        const endTime = Date.now();
+        const duration = Math.round((endTime - startTime) / 1000 * 100) / 100;
+        
+        setProgress({
+          phase: 'completed',
+          current: 1,
+          total: 1,
+          details: {
+            duration: duration,
+            successCount: 1,
+            failureCount: 0,
+            completionRate: 100,
+            totalAttempted: 1,
+            summary: `Scanned webpage in ${duration}s`
+          }
+        });
+        
+        setSuccessMessage(`Website scan completed in ${duration}s, but no vulnerabilities reported.`);
       }
     } catch (err) {
       console.error('Website scan error:', err);

src/lib/RepositoryCrawler.js

@@ -298,13 +298,33 @@ export class RepositoryCrawler {
       // Get file contents with semaphore-controlled concurrency
       const blobFiles = filteredTree.filter(item => item.type === 'blob');
       const filesWithContent = [];
-      let processedFiles = 0;
       const totalFiles = blobFiles.length;
 
       this.clearErrors(); // Clear any previous errors
+      
+      // Track timing and statistics
+      const startTime = Date.now();
+      let successCount = 0;
+      let failureCount = 0;
+      
+      // Initialize progress tracking
+      if (onProgress) {
+        onProgress({
+          phase: 'fetching',
+          current: 0,
+          total: totalFiles,
+          details: { 
+            status: 'Starting file downloads...',
+            startTime: startTime
+          }
+        });
+      }
+      
+      // Track progress with atomic counter to avoid race conditions
+      let completedCount = 0;
 
       // Use semaphore to control concurrency
-      const filePromises = blobFiles.map(file => 
+      const filePromises = blobFiles.map((file, index) => 
         this.semaphore.execute(async () => {
           try {
             // Use raw content URL for better performance
@@ -319,17 +339,40 @@ export class RepositoryCrawler {
                 statusText: response.statusText,
                 filePath: file.path
               });
+              
+              // Update progress and failure count atomically
+              completedCount++;
+              failureCount++;
+              if (onProgress) {
+                onProgress({ 
+                  phase: 'fetching',
+                  current: completedCount, 
+                  total: totalFiles,
+                  details: { 
+                    currentFile: file.path,
+                    successCount: successCount,
+                    failureCount: failureCount
+                  }
+                });
+              }
               return null;
             }
 
             const content = await response.text();
-            processedFiles++;
 
+            // Update progress and success count atomically
+            completedCount++;
+            successCount++;
             if (onProgress) {
               onProgress({ 
                 phase: 'fetching',
-                current: processedFiles, 
-                total: totalFiles
+                current: completedCount, 
+                total: totalFiles,
+                details: { 
+                  currentFile: file.path,
+                  successCount: successCount,
+                  failureCount: failureCount
+                }
               });
             }
 
@@ -339,12 +382,20 @@ export class RepositoryCrawler {
               originalError: error.message,
               filePath: file.path
             });
-            processedFiles++;
+            
+            // Update progress and failure count atomically
+            completedCount++;
+            failureCount++;
             if (onProgress) {
               onProgress({ 
                 phase: 'fetching',
-                current: processedFiles, 
-                total: totalFiles
+                current: completedCount, 
+                total: totalFiles,
+                details: { 
+                  currentFile: file.path,
+                  successCount: successCount,
+                  failureCount: failureCount
+                }
               });
             }
             return null;
@@ -355,21 +406,59 @@ export class RepositoryCrawler {
       const results = await Promise.all(filePromises);
       filesWithContent.push(...results.filter(f => f !== null));
 
-      console.log(`Successfully fetched ${filesWithContent.length} files`);
+      // Calculate final statistics
+      const endTime = Date.now();
+      const duration = Math.round((endTime - startTime) / 1000 * 100) / 100; // seconds with 2 decimal places
+      const actualSuccessCount = filesWithContent.length;
+      const actualFailureCount = totalFiles - actualSuccessCount;
+      const completionRate = totalFiles > 0 ? Math.round((actualSuccessCount / totalFiles) * 100) : 100;
+
+      console.log(`Successfully fetched ${actualSuccessCount} files in ${duration}s`);
 
-      if (this.errors.length > 0) {
-        console.warn(`${this.errors.length} non-fatal errors occurred during file fetching`);
+      if (actualFailureCount > 0) {
+        console.warn(`${actualFailureCount} files failed to download`);
+      }
+      
+      // Send completion summary
+      if (onProgress) {
+        onProgress({
+          phase: 'completed',
+          current: totalFiles,
+          total: totalFiles,
+          details: {
+            duration: duration,
+            successCount: actualSuccessCount,
+            failureCount: actualFailureCount,
+            completionRate: completionRate,
+            totalAttempted: totalFiles,
+            summary: `Scanned ${actualSuccessCount}/${totalFiles} files (${completionRate}%) in ${duration}s`
+          }
+        });
       }
 
       const result = { 
         files: filesWithContent,
         errors: this.errors,
-        partial: this.errors.length > 0
+        partial: actualFailureCount > 0,
+        scanStats: {
+          duration: duration,
+          totalFiles: totalFiles,
+          successCount: actualSuccessCount,
+          failureCount: actualFailureCount,
+          completionRate: completionRate
+        }
       };
       this.cache.set(cacheKey, result, 24 * 60 * 60); // Cache for 24 hours
 
       if (onProgress) {
-        onProgress({ phase: 'analyzing', current: 0, total: filesWithContent.length });
+        onProgress({ 
+          phase: 'analyzing', 
+          current: 0, 
+          total: filesWithContent.length,
+          details: {
+            status: 'Starting vulnerability analysis...'
+          }
+        });
       }
 
       return { ...result, fromCache: false };