Gitea Act Runner 連線負載優化：從 1,300 req/s 降到 170 req/s 的實戰紀錄

1
2
3

Runner → POST /api/actions/runner.v1.RunnerService/FetchTask  → Server (polling)
Runner → POST /api/actions/runner.v1.RunnerService/UpdateLog  → Server (log 回報)
Runner → POST /api/actions/runner.v1.RunnerService/UpdateTask → Server (state 回報)

1

200 runners × 1 req / 2s = 100 req/s

1

200 runners × 3 tasks × 2 req/s = 1,200 req/s

1
2
3
4

type Poller struct {
    consecutiveEmpty  atomic.Int64 // Server 正常回應，但沒有 task
    consecutiveErrors atomic.Int64 // 網路錯誤、超時
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

func (p *Poller) calculateInterval() time.Duration {
    base := p.cfg.Runner.FetchInterval          // 預設 2s
    maxInterval := p.cfg.Runner.FetchIntervalMax // 預設 60s

    n := max(p.consecutiveEmpty.Load(), p.consecutiveErrors.Load())
    if n <= 1 {
        return base
    }

    shift := min(n-1, 5)  // 最大 shift=5，避免 int64 溢位
    interval := base * time.Duration(int64(1)<<shift)
    return min(interval, maxInterval)
}

1
2
3
4
5
6

n=1  → 2s（第一次空回應，不退讓）
n=2  → 4s
n=3  → 8s
n=4  → 16s
n=5  → 32s
n=6+ → 60s（上限）

1
2
3
4
5

func addJitter(d time.Duration) time.Duration {
    jitterRange := int64(d) * 2 / 5  // 40% 的總範圍
    jitter := rand.Int64N(jitterRange) - jitterRange/2  // [-20%, +20%]
    return d + time.Duration(jitter)
}

1
2

limiter := rate.NewLimiter(rate.Every(2*time.Second), 1)
limiter.Wait(ctx)  // 固定速率

1
2
3
4
5
6
7
8
9

// 錯誤：啟動後要等一個完整的 FetchInterval 才能 fetch
func (p *Poller) pollOnce() {
    for {
        timer := time.NewTimer(interval)  // 先等
        <-timer.C
        task, ok := p.fetchTask(ctx)      // 再 fetch
        ...
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

func (p *Poller) pollOnce() {
    for {
        task, ok := p.fetchTask(p.pollingCtx)  // 先 fetch
        if !ok {
            timer := time.NewTimer(interval)   // 沒拿到才 sleep
            select {
            case <-timer.C:
            case <-p.pollingCtx.Done():
                timer.Stop()
                return
            }
            continue
        }
        // 拿到 task，立即執行
        p.runTaskWithRecover(p.jobsCtx, task)
        return
    }
}

1
2
3
4
5

func (r *Reporter) RunDaemon() {
    _ = r.ReportLog(false)    // HTTP call
    _ = r.ReportState(false)  // HTTP call
    time.AfterFunc(time.Second, r.RunDaemon)
}

1
2
3
4
5
6
7

logNotify: make(chan struct{}, 1)

// Fire() 中：
select {
case r.logNotify <- struct{}{}:
default:  // channel 已有通知，不阻塞
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

// 偵測到 step 開始
if step.StartedAt == nil {
    step.StartedAt = timestamppb.New(timestamp)
    urgentState = true  // → 觸發 stateNotify
}

// 偵測到 step 結束
if stepResult, ok := r.parseResult(v); ok {
    step.Result = stepResult
    urgentState = true  // → 觸發 stateNotify
}

1
2
3

if !noMore && len(rows) == 0 {
    return nil  // 不發 HTTP 請求
}

1
2
3
4
5
6
7

// Fire() 中任何狀態變更：
r.stateChanged = true

// ReportState() 中：
if !reportResult && !changed && len(outputs) == 0 {
    return nil  // 連 proto.Clone 都省了
}

1
2
3
4
5
6
7
8

func getHTTPClient(endpoint string, insecure bool) *http.Client {
    if strings.HasPrefix(endpoint, "https://") && insecure {
        return &http.Client{Transport: &http.Transport{
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
        }}
    }
    return http.DefaultClient  // MaxIdleConnsPerHost = 2
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

func getHTTPClient(endpoint string, insecure bool) *http.Client {
    transport := &http.Transport{
        MaxIdleConns:        10,
        MaxIdleConnsPerHost: 10,
        IdleConnTimeout:     90 * time.Second,
    }
    if strings.HasPrefix(endpoint, "https://") && insecure {
        transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
    }
    return &http.Client{Transport: transport}
}

// 共用一個 client
httpClient := getHTTPClient(endpoint, insecure)
PingServiceClient:   pingv1connect.NewPingServiceClient(httpClient, ...)
RunnerServiceClient: runnerv1connect.NewRunnerServiceClient(httpClient, ...)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

runner:
  # Polling backoff
  fetch_interval_max: 60s # 閒置時最大退讓間隔

  # Log 回報
  log_report_interval: 3s # 定時 flush 間隔
  log_report_max_latency: 5s # 單行 log 最大等待時間
  log_report_batch_size: 100 # 觸發立即 flush 的行數

  # State 回報
  state_report_interval: 5s # 定時 flush 間隔（step 轉換仍為即時）