feat(subtitles): batch pre-translation with silent background processing (#838)

* feat: init youtube subtitle

* feat: youtube subtitles

* feat: add youtube subtitles

* feat: youtube subtitles

* revert: revert use draggable

* feat: improve interactive of youtube subtitles

* chore: add changeset

* feat: add config migrate example

* fix: migrate example error

* fix: improve interaction

* fix: fix ai errors

* fix: import error

* chore: remove youtubei

* fix: improve interaction

* refactor: refactor  subtitles.content

* feat: add youtube dont walk element

* fix: interceptor mismatch

* fix: remove --disable-blink-features=AutomationControlled

* fix: remove useless config

* fix: wxt config

* fix: restore LF line endings for use-draggable.ts

* chore: remove unused icon

* fix: some low errors

* fix: toggle button mount error

* fix: extract constant

* feat: move videoSubtitles options to new router

* fix: migrate example error

* refactor: remove Processor class and improve youtube segmenter algorithm

* fix: fix test files

* chore: remove claude settings

* fix: remove useless logger

* fix: improve code quality

* fix: remove useless type

* fix: move standard parser to single files

* feat(subtitles): add zod validation for YouTube subtitle data

* fix: replace setInterval

* feat(subtitles): add error handling with dual display strategy

* fix(subtitles): prevent cleanup() from clearing pending callbacks before resolve

* fix: fix zod schema error

* fix: type check error

* refactor(subtitles): extract error classes for typed error handling

* feat: improve scrolling parser

* feat: add , to SENTENCE_END_PATTERN

* feat: batch translate subtitles

* feat: improve batch translate subtitles

* feat: improve batch strategy

* feat(subtitles): silent background translation with conditional state display

* chore: add changeset

* revert: revert subtitles atom

* feat: improve parser and bolck logic

* chore: remove useless props

* fix: improve code quality

* fix: remove blocks when youtube navigate

* chore: remove useless console log

---------

Co-authored-by: MengXi <[email protected]>

Author: taiiiyang

.changeset/forty-ducks-find.md

@@ -0,0 +1,5 @@
+---
+"@read-frog/extension": patch
+---
+
+feat: Improve the speed of subtitle translation through block translation

src/entrypoints/subtitles.content/atoms.ts

@@ -1,8 +1,10 @@
-import type { StateData, SubtitlesFragment } from '@/utils/subtitles/types'
+import type { StateData, SubtitlesFragment, SubtitlesTranslationBlock } from '@/utils/subtitles/types'
 import { atom, createStore } from 'jotai'
 
 export const subtitlesStore = createStore()
 
+export const subtitlesTranslationBlocksAtom = atom<SubtitlesTranslationBlock[]>([])
+
 export const currentSubtitleAtom = atom<SubtitlesFragment | null>(null)
 
 export const subtitlesStateAtom = atom<StateData | null>(null)

src/entrypoints/subtitles.content/subtitles-scheduler.ts

@@ -1,5 +1,4 @@
 import type { StateData, SubtitlesFragment, SubtitlesState } from '@/utils/subtitles/types'
-import { COMPLETED_STATE_HIDE_DELAY } from '@/utils/constants/subtitles'
 import { currentSubtitleAtom, subtitlesStateAtom, subtitlesStore, subtitlesVisibleAtom } from './atoms'
 
 export class SubtitlesScheduler {
@@ -11,8 +10,6 @@ export class SubtitlesScheduler {
     state: 'idle',
   }
 
-  private hideStateTimeout: NodeJS.Timeout | null = null
-
   constructor({ videoElement }: { videoElement: HTMLVideoElement }) {
     this.videoElement = videoElement
     this.attachListeners()
@@ -28,15 +25,22 @@ export class SubtitlesScheduler {
       return
     }
 
-    this.subtitles.push(...subtitles)
-    this.currentIndex = -1
-    this.updateCurrentSubtitle()
+    const existingStarts = new Set(this.subtitles.map(s => s.start))
+    const newSubtitles = subtitles.filter(s => !existingStarts.has(s.start))
+
+    this.subtitles.push(...newSubtitles)
+    this.subtitles.sort((a, b) => a.start - b.start)
+
+    this.updateSubtitles(this.videoElement.currentTime)
+  }
+
+  getVideoElement(): HTMLVideoElement {
+    return this.videoElement
   }
 
   stop() {
     this.isActive = false
     this.detachListeners()
-    this.clearHideStateTimeout()
     this.updateVisibility()
   }
 
@@ -55,18 +59,7 @@ export class SubtitlesScheduler {
       state,
       message: data?.message,
     }
-    this.clearHideStateTimeout()
-
     this.updateState()
-
-    if (state === 'completed') {
-      this.hideStateTimeout = setTimeout(
-        () => {
-          this.setState('idle')
-        },
-        COMPLETED_STATE_HIDE_DELAY,
-      )
-    }
   }
 
   reset() {
@@ -126,11 +119,4 @@ export class SubtitlesScheduler {
   private updateVisibility() {
     subtitlesStore.set(subtitlesVisibleAtom, this.isActive)
   }
-
-  private clearHideStateTimeout() {
-    if (this.hideStateTimeout) {
-      clearTimeout(this.hideStateTimeout)
-      this.hideStateTimeout = null
-    }
-  }
 }

src/entrypoints/subtitles.content/ui/state-message.tsx

@@ -25,10 +25,6 @@ const STATE_CONFIG: Record<SubtitlesState, { color: string, getText: () => strin
     color: 'oklch(70% 0.19 250)',
     getText: () => i18n.t('subtitles.state.processing'),
   },
-  completed: {
-    color: 'oklch(70% 0.17 165)',
-    getText: () => i18n.t('subtitles.state.completed'),
-  },
   error: {
     color: 'oklch(63% 0.24 25)',
     getText: () => i18n.t('subtitles.state.error'),

src/entrypoints/subtitles.content/universal-adapter.ts

@@ -1,12 +1,14 @@
 import type { PlatformConfig } from '@/entrypoints/subtitles.content/platforms'
 import type { SubtitlesFetcher } from '@/utils/subtitles/fetchers/types'
-import type { SubtitlesFragment } from '@/utils/subtitles/types'
+import type { SubtitlesFragment, SubtitlesTranslationBlock } from '@/utils/subtitles/types'
 import { i18n } from '#imports'
 import { toast } from 'sonner'
 import { HIDE_NATIVE_CAPTIONS_STYLE_ID, NAVIGATION_HANDLER_DELAY, TRANSLATE_BUTTON_CONTAINER_ID } from '@/utils/constants/subtitles'
 import { waitForElement } from '@/utils/dom/wait-for-element'
 import { ToastSubtitlesError } from '@/utils/subtitles/errors'
+import { createSubtitlesBlocks, findNextBlockToTranslate, updateBlockState } from '@/utils/subtitles/processor/block-strategy'
 import { translateSubtitles } from '@/utils/subtitles/processor/translator'
+import { currentSubtitleAtom, subtitlesStore, subtitlesTranslationBlocksAtom } from './atoms'
 import { renderSubtitlesTranslateButton } from './renderer/render-translate-button'
 import { SubtitlesScheduler } from './subtitles-scheduler'
 
@@ -39,12 +41,14 @@ export class UniversalVideoAdapter {
 
   private resetSubtitlesData() {
     this.subtitlesScheduler?.reset()
+    this.stopBlockMonitoring()
     this.originalSubtitles = []
     this.subtitlesFetcher.cleanup()
   }
 
   private resetForNavigation() {
     this.destroyScheduler()
+    this.stopBlockMonitoring()
     this.originalSubtitles = []
     this.cachedVideoId = null
     this.subtitlesFetcher.cleanup()
@@ -194,17 +198,96 @@ export class UniversalVideoAdapter {
     try {
       this.subtitlesScheduler?.setState('processing')
 
-      const translated = await translateSubtitles(this.originalSubtitles)
+      const subtitlesBlocks = createSubtitlesBlocks(this.originalSubtitles)
+      subtitlesStore.set(subtitlesTranslationBlocksAtom, subtitlesBlocks)
 
-      if (this.subtitlesScheduler) {
-        this.subtitlesScheduler.supplementSubtitles(translated)
+      const video = this.subtitlesScheduler?.getVideoElement()
+      const currentTimeMs = (video?.currentTime ?? 0) * 1000
+      const firstBlockToTranslate = findNextBlockToTranslate(subtitlesBlocks, currentTimeMs)
+
+      if (firstBlockToTranslate) {
+        await this.translateSubtitlesBlock(firstBlockToTranslate)
       }
 
-      this.subtitlesScheduler?.setState('completed')
+      this.startBlockMonitoring()
+      this.subtitlesScheduler?.setState('idle')
+    }
+    catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error)
+      this.subtitlesScheduler?.setState('error', { message: errorMessage })
+    }
+  }
+
+  private async translateSubtitlesBlock(batch: SubtitlesTranslationBlock) {
+    const subtitlesBlocks = subtitlesStore.get(subtitlesTranslationBlocksAtom)
+    subtitlesStore.set(subtitlesTranslationBlocksAtom, updateBlockState(subtitlesBlocks, batch.id, 'processing'))
+
+    const currentSubtitle = subtitlesStore.get(currentSubtitleAtom)
+    if (!currentSubtitle) {
+      this.subtitlesScheduler?.setState('processing')
+    }
+
+    try {
+      const translated = await translateSubtitles(batch.fragments)
+
+      const updatedBatches = subtitlesStore.get(subtitlesTranslationBlocksAtom)
+      subtitlesStore.set(
+        subtitlesTranslationBlocksAtom,
+        updateBlockState(updatedBatches, batch.id, 'completed'),
+      )
+
+      this.subtitlesScheduler?.supplementSubtitles(translated)
+      this.subtitlesScheduler?.setState('idle')
     }
     catch (error) {
+      const updatedBatches = subtitlesStore.get(subtitlesTranslationBlocksAtom)
+      subtitlesStore.set(
+        subtitlesTranslationBlocksAtom,
+        updateBlockState(updatedBatches, batch.id, 'error'),
+      )
+
       const errorMessage = error instanceof Error ? error.message : String(error)
       this.subtitlesScheduler?.setState('error', { message: errorMessage })
     }
   }
+
+  private handleBlockCheck = () => {
+    if (!this.subtitlesScheduler)
+      return
+
+    const video = this.subtitlesScheduler.getVideoElement()
+
+    const currentTimeMs = video.currentTime * 1_000
+    const blocks = subtitlesStore.get(subtitlesTranslationBlocksAtom)
+
+    if (blocks.some(b => b.state === 'processing'))
+      return
+
+    const nextBlock = findNextBlockToTranslate(blocks, currentTimeMs)
+    if (nextBlock) {
+      void this.translateSubtitlesBlock(nextBlock)
+    }
+  }
+
+  private startBlockMonitoring() {
+    if (!this.subtitlesScheduler)
+      return
+
+    const video = this.subtitlesScheduler.getVideoElement()
+
+    video.addEventListener('seeked', this.handleBlockCheck)
+    video.addEventListener('timeupdate', this.handleBlockCheck)
+  }
+
+  private stopBlockMonitoring() {
+    subtitlesStore.set(subtitlesTranslationBlocksAtom, [])
+
+    if (!this.subtitlesScheduler)
+      return
+
+    const video = this.subtitlesScheduler.getVideoElement()
+
+    video.removeEventListener('seeked', this.handleBlockCheck)
+    video.removeEventListener('timeupdate', this.handleBlockCheck)
+  }
 }

src/locales/en.yml

@@ -530,7 +530,6 @@ subtitles:
     fetchSuccess: Captions loaded
     fetchFailed: Failed to fetch captions
     processing: Translating captions (approx. 2 minutes)
-    completed: Translation completed
     error: Error occurred
   errors:
     http429: 429 Too many requests to youtube subtitle API, please try again later

src/locales/ja.yml

@@ -526,7 +526,6 @@ subtitles:
     fetchSuccess: 字幕読み込み完了
     fetchFailed: 字幕取得失敗
     processing: 字幕翻訳中（約2分）
-    completed: 翻訳完了
     error: エラーが発生しました
   errors:
     http429: 429 YouTube字幕APIへのリクエストが多すぎます。しばらくしてから再試行してください

src/locales/ko.yml

@@ -527,7 +527,6 @@ subtitles:
     fetchSuccess: 자막 로드 완료
     fetchFailed: 자막 가져오기 실패
     processing: 자막 번역 중 (약 2분 소요)
-    completed: 번역 완료
     error: 오류 발생
   errors:
     http429: 429 유튜브 자막 인터페이스 요청이 너무 많습니다. 잠시 후 다시 시도해 주세요

src/locales/ru.yml

@@ -528,7 +528,6 @@ subtitles:
     fetchSuccess: Субтитры загружены
     fetchFailed: Не удалось получить субтитры
     processing: Перевод субтитров (примерно 2 минуты)
-    completed: Перевод завершён
     error: Произошла ошибка
   errors:
     http429: 429 Слишком много запросов к API YouTube субтитров, пожалуйста, попробуйте снова позже

src/locales/tr.yml

@@ -528,7 +528,6 @@ subtitles:
     fetchSuccess: Altyazılar yüklendi
     fetchFailed: Altyazılar alınamadı
     processing: Altyazılar çevriliyor (yaklaşık 2 dakika)
-    completed: Çeviri tamamlandı
     error: Bir hata oluştu
   errors:
     http429: 429 YouTube altyazı API'sine çok fazla istek gönderildi. Lütfen biraz bekleyip tekrar deneyin.