Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions core/baseline/baseline.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,10 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
}

bl.Raw = append(bl.Header, bl.Body...)
bl.Response, err = pkg.ParseRawResponse(bl.Raw)
if err != nil {
bl.IsValid = false
bl.Reason = pkg.ErrResponseError.Error()
bl.ErrString = err.Error()
return bl
}
if r := bl.Response.Header.Get("Location"); r != "" {
if r := resp.GetHeader("Location"); r != "" {
bl.RedirectURL = r
} else {
bl.RedirectURL = bl.Response.Header.Get("location")
bl.RedirectURL = resp.GetHeader("location")
}

bl.Dir = bl.IsDir()
Expand Down
5 changes: 3 additions & 2 deletions core/pool/brutepool.go
Original file line number Diff line number Diff line change
Expand Up @@ -678,8 +678,9 @@ func (pool *BrutePool) Close() {
// 等待缓存的待处理任务完成
time.Sleep(time.Duration(100) * time.Millisecond)
}
close(pool.additionCh) // 关闭addition管道
//close(pool.checkCh) // 关闭check管道
pool.additionClosed.Store(true)
// additionCh may still have async producers (redirect/crawl/retry/append);
// rely on closeCh/ctx to stop the consumer loop instead of closing the channel.
Comment on lines +681 to +683
Copy link

Copilot AI Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

additionClosed is only set in Close() after the main loop breaks, but there is still a race window where async producers (e.g., redirect/crawl goroutines) can call addAddition after the consumer loop has stopped. Since additionCh is no longer closed and ctx is not canceled on normal completion, those sends can block forever (or enqueue work that will never be drained). Consider canceling pool.ctx in Close() and/or setting additionClosed before closing closeCh (or removing the extra goroutine wrappers around addAddition so the wg.Add happens synchronously).

Copilot uses AI. Check for mistakes.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

如何解决这个潜在的问题呢? 我们之前通过wg去判断是否还有go挂起的没有结束的

pool.Statistor.EndTime = time.Now().Unix()
pool.reqPool.Release()
pool.scopePool.Release()
Expand Down
32 changes: 26 additions & 6 deletions core/pool/pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ type BasePool struct {
ctx context.Context
processCh chan *baseline.Baseline // 待处理的baseline

reqCount int
failedCount int
additionCh chan *Unit
closeCh chan struct{}
wg *sync.WaitGroup
isFallback atomic.Bool
reqCount int
failedCount int
additionCh chan *Unit
closeCh chan struct{}
wg *sync.WaitGroup
isFallback atomic.Bool
additionClosed atomic.Bool
}

func (pool *BasePool) doRetry(bl *baseline.Baseline) {
Expand All @@ -44,12 +45,31 @@ func (pool *BasePool) doRetry(bl *baseline.Baseline) {
}

func (pool *BasePool) addAddition(u *Unit) {
if pool.ctx.Err() != nil || pool.additionClosed.Load() {
return
}

pool.wg.Add(1)
defer func() {
if recover() != nil {
pool.wg.Done()
}
}()

select {
case <-pool.ctx.Done():
pool.wg.Done()
return
case pool.additionCh <- u:
return
default:
// 强行屏蔽报错, 防止goroutine泄露
go func() {
defer func() {
if recover() != nil {
pool.wg.Done()
}
}()
select {
case pool.additionCh <- u:
case <-pool.ctx.Done():
Expand Down