更新版本至1.6.0

muddlelife · muddlelife · commit e26e947aec86 · 2025-01-09T22:38:25.000+08:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,7 +13,12 @@ lazy_static = "1.5.0"
 regex = "1.10.6"
 reqwest = { version = "0.12.7" ,features = ["socks"]}
 serde = { version = "1.0.210", features = ["derive"] }
+serde_json = "1.0.134"
 tokio = { version = "1.40.0", features = ["full"] }
+once_cell = "1.19.0"
+base64 = "0.22.1"
+murmur3 = "0.5.2"
+url = "2.5.4"
 
 [profile.release]
 lto = true # 启用链路时间优化
diff --git a/README.md b/README.md
@@ -1,6 +1,17 @@
 # 说明
 利用Rust编写的高效URL测活工具，主要特点快速、批量、轻量，支持异步。
 
+# 功能介绍
+| 功能  |               描述               |
+|:---:|:------------------------------:|
+| 资产测活 |     对目标资产（域名、IP、URL） 进行测活      |
+| 支持代理 |支持多种代理，包括 HTTP、HTTPS 和 SOCKS 代理 |
+|状态码过滤|   对 HTTP 响应的状态码进行过滤，默认只保留200   |
+|指定路径|       支持指定路径进行测活，默认为根目录        |
+|指纹识别|         采用开源指纹库识别网站CMS         |
+|高并发|          支持高并发请求、支持异步          |
+
+
 # 用法
 ## 帮助信息
 ```text
@@ -16,7 +27,7 @@ Options:
   -c, --status-code <STATUS_CODE>  Display the specified status code [default: 200]
   -p, --path <PATH>                Designated path scan [default: ]
   -x, --proxy <PROXY>              Supported Proxy socks5, http, and https, Example: -x socks5://127.0.0.1:1080
-  -o, --output <OUTPUT>            Output is an csv document, Example: -o result.csv 
+  -o, --output <OUTPUT>            Output is an csv document, Example: -o result.csv
   -h, --help                       Print help (see more with '--help')
   -V, --version                    Print version
 ```
@@ -28,6 +39,7 @@ Options:
 * -c --status-code 显示指定的状态码，默认200，可以输入多个，用逗号隔开，如200,403
 * -p --path 指定扫描路径，默认为空，不指定，如 -p admin
 * -x --proxy 支持代理，目前支持socks5，http，https，如：-x socks5://127.0.0.1:1080
+* -o --output 输出为csv文件，如：-o result.csv
 * -h --help 显示帮助信息
 * -V --version 显示版本信息
 
@@ -58,6 +70,6 @@ windfire -f urls.txt -o result.csv
 ```
 ## 默认打印信息
 ```shell
-https://www.baidu.com [200] [百度一下，你就知道] [BWS/1.1] [https://www.baidu.com/] [414219]
+https://www.baidu.com [200] [百度一下，你就知道] [BWS/1.1] [https://www.baidu.com/] [414219] ["CMS"]
 ```
-包括：起始地址（url）、状态码（status_code）、标题（title）、服务器（server）、跳转后地址（jump_url）、响应页面大小（content_length）
+包括：起始地址（url）、状态码（status_code）、标题（title）、服务器（server）、跳转后地址（jump_url）、响应页面大小（content_length）、指纹信息
diff --git a/src/httpclient.rs b/src/httpclient.rs
@@ -1,11 +1,26 @@
-use crate::utils::{get_format_info, ScanInfo};
+use crate::{
+    utils::{get_favicon_url, get_fofa_iconhash, get_format_info},
+    FINGER_DATA,
+};
+use crossbeam::queue::SegQueue;
 use reqwest::{header, Client};
+use serde::Serialize;
 use std::time::Duration;
-use crossbeam::queue::SegQueue;
 
 pub const USER_AGENT: &str =
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0";
 
+#[derive(Debug, Serialize, Clone)]
+pub struct PrintInfo {
+    pub url: String,
+    pub status_code: u16,
+    pub title: String,
+    pub server: String,
+    pub jump_url: String,
+    pub content_length: usize,
+    pub cms: Vec<String>,
+}
+
 // 创建http客户端
 pub fn create_http_client(timeout: usize, proxy: Option<String>) -> Client {
     let mut headers = header::HeaderMap::new();
@@ -63,7 +78,7 @@ pub async fn send_request(
     url: &str,
     u16_vec: Vec<u16>,
     path: &str,
-    seg_queue: &SegQueue<ScanInfo>,
+    seg_queue: &SegQueue<PrintInfo>,
 ) -> Result<String, reqwest::Error> {
     // 解析URL，如果path为空，则默认为/，如果有值，则加上，还需要处理url有没有/
     let url = if path.is_empty() {
@@ -79,7 +94,7 @@ pub async fn send_request(
 
     let response = client.get(url.as_str()).send().await?;
 
-    let scan_info = get_format_info(response,url);
+    let scan_info = get_format_info(response, url);
     let scan_info = scan_info.await;
 
     let url = scan_info.url;
@@ -88,19 +103,66 @@ pub async fn send_request(
     let content_length = scan_info.content_length;
     let server = scan_info.server;
     let jump_url = scan_info.jump_url;
+    let body = scan_info.body;
+    let header = scan_info.header;
+
+    // 处理url 变为 协议 + 域名 + 端口 + /favicon.ico
+    let favicon_url = get_favicon_url(&url).unwrap_or_else(|_| "".to_string());
+
+    // 获取 icon
+    let icon_hash = get_fofa_iconhash(favicon_url, client)
+        .await
+        .unwrap_or_else(|_| "".to_string());
+    let mut cms_list: Vec<String> = Vec::new();
+
+    // 然后进行指纹匹配，如果能匹配到，则返回cms，遍历指纹
+    for finger in &*FINGER_DATA {
+        let method = finger.method.to_string();
+        let location = finger.location.to_string();
+        let keyword = finger.keyword.clone();
+
+        if method == "keyword" {
+            // 说明为关键词匹配
+            if location == "body" {
+                // 说明是body匹配,keyword 词组都在 body中
+                let all_found = keyword.iter().all(|s| body.contains(s));
+                if all_found {
+                    // 说明匹配成功
+                    cms_list.push(finger.cms.to_string());
+                }
+            } else if location == "header" {
+                // 说明是header匹配
+                let all_found = keyword.iter().all(|s| header.contains(s));
+                if all_found {
+                    // 说明匹配成功
+                    cms_list.push(finger.cms.to_string());
+                }
+            }
+        } else if method == "faviconhash" {
+            // 说明是faviconhash匹配
+            if icon_hash == finger.keyword[0] {
+                // 说明匹配成功
+                cms_list.push(finger.cms.to_string());
+            }
+        }
+    }
+
+    // 对 cms_list 进行去重
+    cms_list.dedup();
 
     if u16_vec.contains(&status_code) {
-        seg_queue.push(ScanInfo {
+        seg_queue.push(PrintInfo {
             url: url.to_string(),
             status_code,
             title: title.to_string(),
             server: server.to_string(),
             jump_url: jump_url.to_string(),
             content_length,
+            cms: cms_list.clone(),
         });
         Ok(format!(
-            "{} [{}] [{}] [{}] [{}] [{}]",
-            url, status_code, title, server, jump_url, content_length
+            "{} [{}] [{}] [{}] [{}] [{}] {:?}",
+            url, status_code, title, server, jump_url, content_length, cms_list
         ))
     } else {
         Ok("".to_string())
diff --git a/src/main.rs b/src/main.rs
@@ -1,16 +1,32 @@
-use crate::httpclient::{create_http_client, send_request};
-use crate::utils::{queue_to_csv, read_file, ScanInfo};
+use crate::httpclient::{create_http_client, send_request, PrintInfo};
+use crate::utils::{queue_to_csv, read_file};
 use clap::Parser;
+use crossbeam::queue::SegQueue;
 use futures::future::join_all;
+use once_cell::sync::Lazy;
 use reqwest::Client;
+use serde::Deserialize;
 use std::sync::Arc;
-use crossbeam::queue::SegQueue;
 use tokio::sync::Semaphore;
 use tokio::task;
 
 mod httpclient;
 mod utils;
 
+#[derive(Debug, Deserialize)]
+pub struct Finger {
+    cms: String,
+    method: String,
+    location: String,
+    keyword: Vec<String>,
+}
+
+// 使用 once_cell::Lazy 来延迟初始化静态变量
+static FINGER_DATA: Lazy<Vec<Finger>> = Lazy::new(|| {
+    let json_data = include_str!("finger.json"); // 使用 include_str! 将文件嵌入到程序中
+    serde_json::from_str(json_data).expect("Failed to parse JSON") // 解析 JSON 数据
+});
+
 #[derive(Parser, Debug)]
 #[command(
     version = "1.5.0",
@@ -64,7 +80,7 @@ async fn main() {
 
     let path = args.path;
     let proxy = args.proxy;
-    let seg_queue: Arc<SegQueue<ScanInfo>> = Arc::new(SegQueue::new());
+    let seg_queue: Arc<SegQueue<PrintInfo>> = Arc::new(SegQueue::new());
 
     if let Some(url) = args.url {
         let client = create_http_client(args.timeout, proxy);
@@ -90,11 +106,12 @@ async fn main() {
                     let semaphore = Arc::clone(&semaphore);
                     let client: Client = client.clone();
                     let u16_vec = u16_vec.clone();
-                    let path = path.clone();
+                    let path: String = path.clone();
                     let seg_queue = Arc::clone(&seg_queue);
                     futures.push(task::spawn(async move {
                         let permit = semaphore.acquire().await.unwrap();
-                        let result = send_request(client, url.as_str(), u16_vec, &path, &seg_queue).await;
+                        let result =
+                            send_request(client, url.as_str(), u16_vec, &path, &seg_queue).await;
                         match result {
                             Ok(result) => {
                                 if result != "" {
diff --git a/src/utils.rs b/src/utils.rs
