first commit

backend/api/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY pyproject.toml .
+RUN pip install --no-cache-dir \
+    "fastapi>=0.110.0" \
+    "uvicorn[standard]>=0.29.0" \
+    "httpx>=0.27.0" \
+    "pydantic>=2.6.0" \
+    "apscheduler>=3.10.4"
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/api/__init__.py

@@ -0,0 +1 @@
+# Backend Python packages init

backend/api/main.py

@@ -0,0 +1,192 @@
+"""
+API FastAPI — Veille YouTube Kubernetes.
+"""
+
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from fastapi import FastAPI, Query, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+
+from .models import Video, VideoList, RefreshResult, RefreshRequest
+from .storage import load_videos, save_videos, get_last_updated, load_config, save_config, load_quota_status, save_quota_status
+from .youtube_client import fetch_all_videos, QuotaExceededError
+from scoring.scorer import score_video
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+app = FastAPI(
+    title="YTVeille",
+    description="API de veille automatique des meilleures vidéos YouTube en français",
+    version="1.0.0",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+_refresh_running = False
+
+
+def _run_refresh() -> RefreshResult:
+    """Pipeline : fetch → score → persist."""
+    import asyncio
+
+    config = load_config()
+    queries = config.get("queries")
+    raw = asyncio.run(fetch_all_videos(queries))
+    logger.info("Vidéos récupérées : %d", len(raw))
+
+    scored = []
+    for v in raw:
+        s, topics = score_video(v)
+        v["score"] = s
+        v["topics"] = topics
+        scored.append(v)
+
+    # Trier par score décroissant
+    scored.sort(key=lambda x: x["score"], reverse=True)
+    save_videos(scored)
+    logger.info("Vidéos sauvegardées : %d", len(scored))
+
+    return RefreshResult(
+        fetched=len(raw),
+        scored=len(scored),
+        stored=len(scored),
+        timestamp=datetime.now(timezone.utc),
+    )
+
+
+@app.get("/api/videos", response_model=VideoList)
+def list_videos(
+    q: Optional[str] = Query(None),
+    min_score: float = Query(0.0, ge=0, le=100),
+    topic: Optional[str] = Query(None),
+    source_query: list[str] = Query(default=[]),
+    days: int = Query(30, ge=1, le=90),
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+):
+    """Liste paginée des vidéos avec filtres."""
+    all_videos = load_videos()
+
+    # Filtre texte (titre, chaîne, tags YouTube)
+    if q:
+        q_lower = q.lower()
+        all_videos = [
+            v for v in all_videos
+            if q_lower in v.get("title", "").lower()
+            or q_lower in v.get("channel", "").lower()
+            or any(q_lower in t.lower() for t in v.get("tags", []))
+        ]
+
+    # Filtre par requête source (OR entre les requêtes sélectionnées)
+    if source_query:
+        all_videos = [
+            v for v in all_videos
+            if any(sq in v.get("source_queries", []) for sq in source_query)
+        ]
+
+    # Filtre score
+    filtered = [v for v in all_videos if v.get("score", 0) >= min_score]
+
+    # Filtre topic
+    if topic:
+        filtered = [v for v in filtered if topic in v.get("topics", [])]
+
+    # Filtre date
+    from datetime import timedelta
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    result = []
+    for v in filtered:
+        pub = v.get("published_at", "")
+        try:
+            dt = datetime.fromisoformat(str(pub).replace("Z", "+00:00"))
+            if dt >= cutoff:
+                result.append(v)
+        except Exception:
+            result.append(v)
+
+    total = len(result)
+    start = (page - 1) * page_size
+    page_items = result[start : start + page_size]
+
+    return VideoList(
+        total=total,
+        page=page,
+        page_size=page_size,
+        items=[Video(**v) for v in page_items],
+    )
+
+
+@app.get("/api/videos/{video_id}", response_model=Video)
+def get_video(video_id: str):
+    """Détail d'une vidéo par ID."""
+    all_videos = load_videos()
+    for v in all_videos:
+        if v["id"] == video_id:
+            return Video(**v)
+    raise HTTPException(status_code=404, detail="Vidéo non trouvée")
+
+
+@app.get("/api/config")
+def get_config():
+    """Retourne la configuration de recherche actuelle."""
+    return load_config()
+
+
+@app.post("/api/refresh", response_model=RefreshResult)
+def refresh(background_tasks: BackgroundTasks, body: Optional[RefreshRequest] = None):
+    """Déclenche une mise à jour. Si body.queries fourni, sauvegarde la config."""
+    global _refresh_running
+    if _refresh_running:
+        raise HTTPException(status_code=409, detail="Un refresh est déjà en cours")
+    if body and body.queries:
+        save_config({"queries": body.queries})
+    _refresh_running = True
+
+    def _wrapped():
+        global _refresh_running
+        try:
+            _run_refresh()
+            save_quota_status(False)
+        except QuotaExceededError:
+            save_quota_status(True)
+            logger.warning("Quota YouTube API dépassé — refresh abandonné")
+        except Exception as exc:
+            logger.error("Erreur lors du refresh : %s", exc)
+        finally:
+            _refresh_running = False
+
+    background_tasks.add_task(_wrapped)
+    return RefreshResult(
+        fetched=0,
+        scored=0,
+        stored=0,
+        timestamp=datetime.now(timezone.utc),
+    )
+
+
+@app.get("/api/status")
+def status():
+    """État de l'API et date de dernière mise à jour."""
+    last = get_last_updated()
+    videos = load_videos()
+    quota = load_quota_status()
+    return {
+        "status": "ok",
+        "video_count": len(videos),
+        "last_updated": last.isoformat() if last else None,
+        "refresh_running": _refresh_running,
+        "queries": load_config().get("queries", []),
+        "quota_exceeded": quota.get("exceeded", False),
+        "quota_exceeded_at": quota.get("exceeded_at"),
+    }

backend/api/models.py

@@ -0,0 +1,46 @@
+from pydantic import BaseModel
+from typing import List, Optional
+from datetime import datetime
+
+
+class Video(BaseModel):
+    id: str
+    title: str
+    channel: str
+    published_at: datetime
+    duration_seconds: int
+    view_count: int
+    like_count: int
+    thumbnail_url: str
+    youtube_url: str
+    tags: List[str] = []
+    has_chapters: bool = False
+    score: float = 0.0
+    topics: List[str] = []
+    source_queries: List[str] = []
+
+
+class VideoList(BaseModel):
+    total: int
+    page: int
+    page_size: int
+    items: List[Video]
+
+
+class FilterParams(BaseModel):
+    min_score: float = 0.0
+    topic: Optional[str] = None
+    days: int = 30
+    page: int = 1
+    page_size: int = 20
+
+
+class RefreshRequest(BaseModel):
+    queries: Optional[List[str]] = None
+
+
+class RefreshResult(BaseModel):
+    fetched: int
+    scored: int
+    stored: int
+    timestamp: datetime

backend/api/storage.py

@@ -0,0 +1,92 @@
+"""
+Persistance des vidéos dans un fichier JSON.
+Écriture atomique pour éviter la corruption.
+"""
+
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+DATA_PATH = Path(os.environ.get("DATA_PATH", "/app/data/videos.json"))
+CONFIG_PATH = DATA_PATH.parent / "config.json"
+QUOTA_PATH = DATA_PATH.parent / "quota_status.json"
+
+DEFAULT_QUERIES = [
+    "Kubernetes production français",
+    "Kubernetes architecture français",
+    "Kubernetes retour d'expérience",
+    "Kubernetes incident production français",
+    "Kubernetes scaling français",
+    "Kubernetes observabilité",
+    "Kubernetes tutoriel français",
+    "Kubernetes déploiement français",
+]
+
+
+def _ensure_dir() -> None:
+    DATA_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+
+def load_quota_status() -> dict:
+    """Charge l'état du quota YouTube API."""
+    if not QUOTA_PATH.exists():
+        return {"exceeded": False, "exceeded_at": None}
+    with QUOTA_PATH.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_quota_status(exceeded: bool) -> None:
+    """Persiste l'état du quota YouTube API."""
+    _ensure_dir()
+    tmp = QUOTA_PATH.with_suffix(".tmp")
+    data = {
+        "exceeded": exceeded,
+        "exceeded_at": datetime.now(timezone.utc).isoformat() if exceeded else None,
+    }
+    with tmp.open("w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False)
+    tmp.replace(QUOTA_PATH)
+
+
+def load_config() -> dict:
+    """Charge la configuration de recherche (queries)."""
+    if not CONFIG_PATH.exists():
+        return {"queries": DEFAULT_QUERIES}
+    with CONFIG_PATH.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_config(config: dict) -> None:
+    """Sauvegarde la configuration de recherche."""
+    _ensure_dir()
+    tmp = CONFIG_PATH.with_suffix(".tmp")
+    with tmp.open("w", encoding="utf-8") as f:
+        json.dump(config, f, ensure_ascii=False, indent=2)
+    tmp.replace(CONFIG_PATH)
+
+
+def load_videos() -> list[dict[str, Any]]:
+    """Charge la liste des vidéos depuis le fichier JSON."""
+    if not DATA_PATH.exists():
+        return []
+    with DATA_PATH.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_videos(videos: list[dict[str, Any]]) -> None:
+    """Sauvegarde atomique des vidéos (écriture via fichier temporaire)."""
+    _ensure_dir()
+    tmp = DATA_PATH.with_suffix(".tmp")
+    with tmp.open("w", encoding="utf-8") as f:
+        json.dump(videos, f, ensure_ascii=False, indent=2, default=str)
+    tmp.replace(DATA_PATH)
+
+
+def get_last_updated() -> datetime | None:
+    """Retourne la date de dernière modification du fichier de données."""
+    if not DATA_PATH.exists():
+        return None
+    return datetime.fromtimestamp(DATA_PATH.stat().st_mtime)

backend/api/youtube_client.py

@@ -0,0 +1,243 @@
+"""
+Client YouTube Data API v3.
+Recherche multi-mots-clés de vidéos Kubernetes en français.
+"""
+
+import os
+import re
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class QuotaExceededError(Exception):
+    """Levée quand le quota journalier YouTube API est dépassé (HTTP 403)."""
+
+YOUTUBE_API_BASE = "https://www.googleapis.com/youtube/v3"
+
+SEARCH_QUERIES = [
+    "Kubernetes production français",
+    "Kubernetes architecture français",
+    "Kubernetes retour d'expérience",
+    "Kubernetes incident production français",
+    "Kubernetes scaling français",
+    "Kubernetes observabilité",
+    "Kubernetes tutoriel français",
+    "Kubernetes déploiement français",
+]
+
+
+def _get_api_key() -> str:
+    key = os.environ.get("YOUTUBE_API_KEY", "")
+    if not key:
+        raise RuntimeError("YOUTUBE_API_KEY manquante dans les variables d'environnement")
+    return key
+
+
+def _iso_days_ago(days: int = 90) -> str:
+    dt = datetime.now(timezone.utc) - timedelta(days=days)
+    return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _parse_duration(iso_duration: str) -> int:
+    """Convertit ISO 8601 duration (PT1H2M3S) en secondes."""
+    pattern = r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?"
+    m = re.match(pattern, iso_duration or "")
+    if not m:
+        return 0
+    h = int(m.group(1) or 0)
+    mn = int(m.group(2) or 0)
+    s = int(m.group(3) or 0)
+    return h * 3600 + mn * 60 + s
+
+
+def _has_chapters(description: str) -> bool:
+    """Détecte la présence de chapitrage (timestamps 0:00 dans la description)."""
+    return bool(re.search(r"^\s*\d+:\d+", description or "", re.MULTILINE))
+
+
+_FRENCH_WORDS = {
+    "avec", "pour", "dans", "sur", "cette", "votre", "notre", "mais", "très",
+    "comme", "nous", "vous", "comment", "voici", "voilà", "pourquoi", "mise",
+    "une", "des", "les", "est", "par", "qui", "que", "aux", "tout", "son",
+    "leur", "aussi", "bien", "chez", "vers", "sous", "avoir", "être", "faire",
+}
+
+_FR_ACCENT_RE = re.compile(r"[éèêëàâùûîïôçœæ]", re.IGNORECASE)
+
+
+def _is_likely_french(snippet: dict) -> bool:
+    """Retourne True si la vidéo est vraisemblablement en français."""
+    audio_lang = (snippet.get("defaultAudioLanguage") or "").lower()
+    default_lang = (snippet.get("defaultLanguage") or "").lower()
+    title = snippet.get("title", "")
+    first_desc = (snippet.get("description") or "")[:400]
+
+    # Langue explicitement déclarée française
+    if audio_lang.startswith("fr") or default_lang.startswith("fr"):
+        return True
+    # Accents français dans le titre → signal fort
+    if _FR_ACCENT_RE.search(title):
+        return True
+    # Langue explicitement anglaise (ou autre non-fr) → rejeter
+    if audio_lang and not audio_lang.startswith("fr"):
+        return False
+    if default_lang and not default_lang.startswith("fr"):
+        return False
+    # Langue inconnue : détecter via accents ou mots français dans la description
+    if _FR_ACCENT_RE.search(first_desc):
+        return True
+    words = set(f"{title} {first_desc}".lower().split())
+    return bool(words & _FRENCH_WORDS)
+
+
+_STOPWORDS = {
+    "en", "de", "du", "le", "la", "les", "un", "une", "des", "et", "ou",
+    "pour", "sur", "avec", "dans", "par", "au", "aux", "ce", "qui", "que",
+    "français", "french", "fr", "how", "to", "the", "and", "with",
+}
+
+
+def _extract_keywords(query: str) -> list[str]:
+    """Extrait les mots-clés significatifs d'une requête (mots > 2 chars, hors stopwords)."""
+    return [w for w in query.lower().split() if len(w) > 2 and w not in _STOPWORDS]
+
+
+def _is_relevant(video: dict, source_queries: list[str]) -> bool:
+    """Retourne True si au moins un mot-clé de la requête source est dans le titre, les tags ou la description."""
+    title = video.get("title", "").lower()
+    tags = " ".join(video.get("tags", [])).lower()
+    desc = video.get("_desc_preview", "").lower()
+    text = f"{title} {tags} {desc}"
+    for query in source_queries:
+        for kw in _extract_keywords(query):
+            if kw in text:
+                return True
+    return False
+
+
+def _add_french_hint(query: str) -> str:
+    """Ajoute 'français' à la requête si aucun indicateur de langue n'est présent."""
+    lower = query.lower()
+    if any(w in lower for w in ["français", "francais", "french", " fr "]):
+        return query
+    return f"{query} français"
+
+
+async def search_videos(query: str, client: httpx.AsyncClient, api_key: str) -> list[str]:
+    """Retourne une liste d'IDs vidéos pour une requête donnée."""
+    params = {
+        "part": "id",
+        "q": _add_french_hint(query),
+        "type": "video",
+        "relevanceLanguage": "fr",
+        "regionCode": "FR",
+        "publishedAfter": _iso_days_ago(90),
+        "maxResults": 25,
+        "key": api_key,
+    }
+    resp = await client.get(f"{YOUTUBE_API_BASE}/search", params=params, timeout=15.0)
+    try:
+        resp.raise_for_status()
+    except httpx.HTTPStatusError as e:
+        if e.response.status_code == 403:
+            raise QuotaExceededError("Quota YouTube API journalier dépassé (HTTP 403)") from e
+        raise
+    data = resp.json()
+    return [item["id"]["videoId"] for item in data.get("items", [])]
+
+
+async def get_video_details(video_ids: list[str], client: httpx.AsyncClient, api_key: str) -> list[dict[str, Any]]:
+    """Retourne les détails enrichis pour une liste d'IDs vidéos."""
+    if not video_ids:
+        return []
+
+    # Batches de 50 (limite API)
+    results = []
+    for i in range(0, len(video_ids), 50):
+        batch = video_ids[i : i + 50]
+        params = {
+            "part": "snippet,contentDetails,statistics",
+            "id": ",".join(batch),
+            "key": api_key,
+        }
+        resp = await client.get(f"{YOUTUBE_API_BASE}/videos", params=params, timeout=15.0)
+        resp.raise_for_status()
+        results.extend(resp.json().get("items", []))
+
+    videos = []
+    for item in results:
+        snippet = item.get("snippet", {})
+        stats = item.get("statistics", {})
+        details = item.get("contentDetails", {})
+
+        duration_s = _parse_duration(details.get("duration", ""))
+        description = snippet.get("description", "")
+
+        if not _is_likely_french(snippet):
+            continue
+
+        title = snippet.get("title", "")
+        videos.append({
+            "id": item["id"],
+            "title": title,
+            "channel": snippet.get("channelTitle", ""),
+            "published_at": snippet.get("publishedAt", ""),
+            "duration_seconds": duration_s,
+            "view_count": int(stats.get("viewCount", 0)),
+            "like_count": int(stats.get("likeCount", 0)),
+            "thumbnail_url": snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
+            "youtube_url": f"https://www.youtube.com/watch?v={item['id']}",
+            "tags": snippet.get("tags", [])[:20],
+            "has_chapters": _has_chapters(description),
+            "_desc_preview": description[:600],  # temporaire, nettoyé après filtrage
+        })
+
+    return videos
+
+
+async def fetch_all_videos(queries: list[str] | None = None) -> list[dict[str, Any]]:
+    """Lance la recherche sur tous les mots-clés, déduplique par ID et tracke les requêtes sources."""
+    if queries is None:
+        queries = SEARCH_QUERIES
+    api_key = _get_api_key()
+
+    # Map video_id → liste des requêtes qui l'ont renvoyé
+    id_to_queries: dict[str, list[str]] = {}
+
+    async with httpx.AsyncClient() as client:
+        for query in queries:
+            try:
+                ids = await search_videos(query, client, api_key)
+                for vid_id in ids:
+                    id_to_queries.setdefault(vid_id, []).append(query)
+                logger.info("Requête '%s' → %d IDs", query, len(ids))
+            except QuotaExceededError:
+                logger.warning("Quota YouTube dépassé — arrêt des requêtes restantes")
+                raise
+            except Exception as exc:
+                logger.error("Erreur pour la requête '%s': %s", query, exc)
+
+        if not id_to_queries:
+            return []
+
+        all_videos = await get_video_details(list(id_to_queries.keys()), client, api_key)
+
+    # Attacher les requêtes sources à chaque vidéo
+    for video in all_videos:
+        video["source_queries"] = id_to_queries.get(video["id"], [])
+
+    # Filtrer les vidéos non pertinentes (titre/tags/description sans aucun mot-clé de la requête)
+    before = len(all_videos)
+    all_videos = [v for v in all_videos if _is_relevant(v, v["source_queries"])]
+    logger.info("Filtre pertinence : %d → %d vidéos", before, len(all_videos))
+
+    # Supprimer le champ temporaire avant persistance
+    for v in all_videos:
+        v.pop("_desc_preview", None)
+
+    return all_videos