Release standalone PPT template workflow 0.5.0
This commit is contained in:
@@ -0,0 +1,564 @@
|
||||
"""Shared primitives for web image providers.
|
||||
|
||||
This module is the single home for everything that all four providers
|
||||
(Openverse / Wikimedia / Pexels / Pixabay) need:
|
||||
|
||||
- License tier classification (the central abstraction of this module)
|
||||
- Search request / asset candidate dataclasses
|
||||
- Query simplification for keyword-based image APIs
|
||||
- Candidate scoring
|
||||
- Attribution text builder
|
||||
- Small helpers (orientation, json path, etc.)
|
||||
|
||||
Provider-specific code (API URLs, payload shape, parse_results) lives in
|
||||
the corresponding provider_<name>.py module and only imports from here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("This is an internal helper module used by image_search.py and the four web image providers.")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project-wide constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
USER_AGENT = "PPTMaster/1.0 (https://github.com/hugohe3/ppt-master)"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# License tier classification
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Every accepted candidate is classified into exactly one of two tiers:
|
||||
#
|
||||
# "no-attribution" -> No on-slide credit needed (CC0, PD, Pexels,
|
||||
# Pixabay). Default search target.
|
||||
# "attribution-required" -> CC BY / CC BY-SA. Executor must add an
|
||||
# inline credit text element on the slide.
|
||||
#
|
||||
# Anything else (CC BY-NC, CC BY-ND, all-rights-reserved, unknown) returns
|
||||
# None and the candidate is rejected outright.
|
||||
|
||||
LICENSE_TIER_NO_ATTRIBUTION = "no-attribution"
|
||||
LICENSE_TIER_ATTRIBUTION_REQUIRED = "attribution-required"
|
||||
|
||||
# Tokens that mark a license as "no attribution required".
|
||||
NO_ATTRIBUTION_TOKENS: tuple[str, ...] = (
|
||||
"cc0",
|
||||
"public domain",
|
||||
"publicdomain",
|
||||
"creativecommons.org/publicdomain/",
|
||||
"pexels license",
|
||||
"pixabay content license",
|
||||
"pixabay license",
|
||||
)
|
||||
|
||||
# Tokens that mark a license as "attribution required".
|
||||
ATTRIBUTION_REQUIRED_TOKENS: tuple[str, ...] = (
|
||||
"cc by",
|
||||
"cc-by",
|
||||
"by-sa",
|
||||
"by sa",
|
||||
"creativecommons.org/licenses/by/",
|
||||
"creativecommons.org/licenses/by-sa/",
|
||||
)
|
||||
|
||||
# Tokens that disqualify a candidate entirely.
|
||||
REJECTED_TOKENS: tuple[str, ...] = (
|
||||
"by-nc",
|
||||
"by nc",
|
||||
"noncommercial",
|
||||
"non-commercial",
|
||||
"by-nd",
|
||||
"by nd",
|
||||
"no derivatives",
|
||||
"noderivatives",
|
||||
"all rights reserved",
|
||||
)
|
||||
|
||||
|
||||
# Canonical display forms for license names. Different providers report
|
||||
# the same license with different capitalization (Openverse: "cc0",
|
||||
# Wikimedia: "Public domain"); the Executor renders these as on-slide
|
||||
# text, so a normalized form prevents inconsistent credits.
|
||||
_LICENSE_NAME_CANON: dict[str, str] = {
|
||||
"cc0": "CC0",
|
||||
"cc 0": "CC0",
|
||||
"public domain": "Public Domain",
|
||||
"publicdomain": "Public Domain",
|
||||
"pdm": "Public Domain",
|
||||
"pexels license": "Pexels License",
|
||||
"pixabay content license": "Pixabay Content License",
|
||||
"pixabay license": "Pixabay Content License",
|
||||
}
|
||||
|
||||
# CC license short-name pattern used to canonicalize "cc by 4.0" → "CC BY 4.0".
|
||||
_CC_PATTERN = re.compile(
|
||||
r"^\s*cc[\s-]+(by(?:[\s-]+(?:sa|nc|nd))*)\s*([0-9.]*)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_license_name(name: str) -> str:
|
||||
"""Return a canonical display form for a license name.
|
||||
|
||||
Maps common aliases to a consistent capitalization so the on-slide
|
||||
credit text written by the Executor is uniform across providers.
|
||||
Unknown inputs are returned trimmed but otherwise unchanged.
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
key = name.strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
|
||||
if key in _LICENSE_NAME_CANON:
|
||||
return _LICENSE_NAME_CANON[key]
|
||||
|
||||
cc_match = _CC_PATTERN.match(key)
|
||||
if cc_match:
|
||||
suffix_raw, version = cc_match.group(1), cc_match.group(2)
|
||||
suffix = suffix_raw.replace(" ", "-").upper()
|
||||
return f"CC {suffix} {version}".strip()
|
||||
|
||||
return name.strip()
|
||||
|
||||
|
||||
def classify_license(
|
||||
license_name: str,
|
||||
license_url: str = "",
|
||||
provider: str = "",
|
||||
) -> Optional[str]:
|
||||
"""Classify a license string into one of the two tiers, or reject it.
|
||||
|
||||
Returns:
|
||||
``"no-attribution"`` / ``"attribution-required"`` / ``None``.
|
||||
|
||||
The provider hint lets us treat Pexels and Pixabay's own licenses as
|
||||
``no-attribution`` even when the upstream API only returns a short
|
||||
label like ``"Pexels"``.
|
||||
"""
|
||||
text = " ".join(
|
||||
part.strip().lower()
|
||||
for part in (license_name or "", license_url or "")
|
||||
if part
|
||||
)
|
||||
provider_key = (provider or "").strip().lower()
|
||||
|
||||
if not text and not provider_key:
|
||||
return None
|
||||
|
||||
if any(token in text for token in REJECTED_TOKENS):
|
||||
return None
|
||||
|
||||
if any(token in text for token in NO_ATTRIBUTION_TOKENS):
|
||||
return LICENSE_TIER_NO_ATTRIBUTION
|
||||
|
||||
# Provider-default fallback: pexels / pixabay items often arrive with a
|
||||
# bare "Pexels" / "Pixabay" license string. Their site-wide license is
|
||||
# "free for commercial use, no attribution required".
|
||||
#
|
||||
# Guard: require the license text to actually mention the provider name,
|
||||
# so an empty / missing license field never silently passes as no-attribution.
|
||||
if (
|
||||
provider_key in {"pexels", "pixabay"}
|
||||
and provider_key in text
|
||||
and not any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS)
|
||||
):
|
||||
return LICENSE_TIER_NO_ATTRIBUTION
|
||||
|
||||
if any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS):
|
||||
return LICENSE_TIER_ATTRIBUTION_REQUIRED
|
||||
|
||||
return None # unknown license -> reject
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageSearchRequest:
|
||||
"""A single image search intent passed to a provider."""
|
||||
|
||||
query: str
|
||||
purpose: str = ""
|
||||
orientation: str = "" # "landscape" / "portrait" / "square" / ""
|
||||
min_width: int = 0
|
||||
min_height: int = 0
|
||||
filename: str = ""
|
||||
slide: str = ""
|
||||
required_terms: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetCandidate:
|
||||
"""One ranked candidate returned by a provider's parse_results."""
|
||||
|
||||
provider: str
|
||||
title: str
|
||||
asset_id: str = ""
|
||||
source_page_url: str = ""
|
||||
license_name: str = ""
|
||||
license_url: str = ""
|
||||
license_tier: str = "" # one of LICENSE_TIER_* constants
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
download_url: str = ""
|
||||
author: str = ""
|
||||
raw: Any = field(default=None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query simplification
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Web image APIs do keyword matching against image metadata, not semantic
|
||||
# search. Long, descriptive queries with brand names, HEX codes, and
|
||||
# composition notes return zero results. We progressively trim the query
|
||||
# down to the most concrete nouns.
|
||||
|
||||
_NOISE_WORDS = frozenset({
|
||||
# Brand / product names
|
||||
"claude", "openai", "gpt", "gemini", "copilot", "chatgpt", "midjourney",
|
||||
"stable", "diffusion", "dall-e", "cursor", "anthropic", "microsoft",
|
||||
"google", "apple", "meta", "nvidia", "tesla",
|
||||
# Generic filler
|
||||
"using", "with", "from", "that", "this", "have", "been", "will",
|
||||
"into", "more", "also", "very", "some", "than", "them", "other",
|
||||
})
|
||||
|
||||
# Words that look generic but are actually useful when they ARE the
|
||||
# subject of the deck (e.g. a deck about AI). We only drop them when
|
||||
# there are still other concrete nouns left.
|
||||
_SOFT_NOISE_WORDS = frozenset({
|
||||
"ai", "code", "software", "system", "digital", "platform", "solution",
|
||||
"application", "interface", "framework", "algorithm", "api", "sdk",
|
||||
"assistant", "tool", "service", "technology", "tech", "program",
|
||||
# Visual-quality / usage terms. These are helpful in the full provider
|
||||
# query, but should not consume the 3-4 keyword fallback budget or
|
||||
# dominate relevance scoring over the real subject.
|
||||
"professional", "editorial", "commercial", "premium", "stock",
|
||||
"photo", "photograph", "photography", "image", "picture", "visual",
|
||||
"background", "hero", "cover", "banner", "wallpaper",
|
||||
"high", "quality", "resolution", "sharp", "clean", "cinematic",
|
||||
"dramatic", "lighting", "light", "modern", "natural", "visible",
|
||||
})
|
||||
|
||||
_TOKEN_STRIP_CHARS = ".,;:!?\"'()[]{}，。；：！？、"
|
||||
_MATCH_SEPARATOR_RE = re.compile(r"""[\s\-_./:;,'"()[\]{}]+""")
|
||||
|
||||
|
||||
def simplify_query(query: str, max_words: int = 4) -> str:
|
||||
"""Trim a verbose query into a short keyword phrase.
|
||||
|
||||
Strategy:
|
||||
1. Strip HEX color codes and parenthetical asides.
|
||||
2. Drop hard-noise words (brand names, generic filler).
|
||||
3. Drop soft-noise words ONLY if concrete nouns remain.
|
||||
4. If the result would be empty, return the original query
|
||||
(fail-open: better an over-broad search than zero results).
|
||||
5. Cap at ``max_words`` words.
|
||||
"""
|
||||
cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query)
|
||||
cleaned = re.sub(r"\([^)]*\)", "", cleaned)
|
||||
words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
|
||||
words = [w for w in words if len(w) > 2]
|
||||
|
||||
after_hard = [w for w in words if w.lower() not in _NOISE_WORDS]
|
||||
after_soft = [w for w in after_hard if w.lower() not in _SOFT_NOISE_WORDS]
|
||||
|
||||
# Only drop soft-noise if there are still concrete nouns left.
|
||||
filtered = after_soft if after_soft else after_hard
|
||||
|
||||
if not filtered:
|
||||
# Everything got filtered. Fail open: return the original query.
|
||||
return query.strip()
|
||||
|
||||
return " ".join(filtered[:max_words])
|
||||
|
||||
|
||||
def build_query_progression(query: str) -> list[str]:
|
||||
"""Return a list of progressively simpler queries to try in order.
|
||||
|
||||
Stops as soon as one of them yields candidates upstream. Duplicates
|
||||
are dropped while preserving order.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for candidate in (
|
||||
query,
|
||||
simplify_query(query, max_words=4),
|
||||
simplify_query(query, max_words=3),
|
||||
simplify_query(query, max_words=2),
|
||||
simplify_query(query, max_words=1),
|
||||
):
|
||||
candidate = candidate.strip()
|
||||
if candidate and candidate not in seen:
|
||||
seen.add(candidate)
|
||||
out.append(candidate)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normalize_orientation(width: int, height: int) -> str:
|
||||
if width <= 0 or height <= 0:
|
||||
return "unknown"
|
||||
if width > height:
|
||||
return "landscape"
|
||||
if height > width:
|
||||
return "portrait"
|
||||
return "square"
|
||||
|
||||
|
||||
def _query_tokens(query: str) -> list[str]:
|
||||
"""Extract ASCII keyword tokens from a query for relevance scoring.
|
||||
|
||||
Uses the same noise-word filtering as ``simplify_query`` so the
|
||||
relevance signal lines up with the keywords we actually search by.
|
||||
Non-ASCII tokens (CJK etc.) are dropped — image metadata is mostly
|
||||
English even on multi-language providers, so substring matching CJK
|
||||
against an English title is unreliable. When this leaves no tokens,
|
||||
``compute_relevance`` falls back to neutral (1.0) and lets the other
|
||||
score dimensions decide.
|
||||
"""
|
||||
cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query.lower())
|
||||
cleaned = re.sub(r"\([^)]*\)", "", cleaned)
|
||||
words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
|
||||
words = [w for w in words if len(w) > 2 and w.isascii()]
|
||||
if not words:
|
||||
return []
|
||||
after_hard = [w for w in words if w not in _NOISE_WORDS]
|
||||
after_soft = [w for w in after_hard if w not in _SOFT_NOISE_WORDS]
|
||||
return after_soft if after_soft else after_hard
|
||||
|
||||
|
||||
def _candidate_text(candidate: AssetCandidate) -> str:
|
||||
"""Concatenate the candidate's matchable metadata fields for scoring."""
|
||||
return " ".join(
|
||||
filter(
|
||||
None,
|
||||
(
|
||||
candidate.title,
|
||||
candidate.author,
|
||||
candidate.source_page_url,
|
||||
),
|
||||
)
|
||||
).lower()
|
||||
|
||||
|
||||
def _normalize_match_text(text: str) -> str:
|
||||
"""Normalize metadata / required terms for conservative substring matching."""
|
||||
lowered = (text or "").lower()
|
||||
return _MATCH_SEPARATOR_RE.sub(" ", lowered).strip()
|
||||
|
||||
|
||||
def _term_group_alternatives(term_group: str) -> list[str]:
|
||||
"""Split one required term group into alternatives.
|
||||
|
||||
``"Jiefangbei|Liberation Monument"`` means either alternative satisfies
|
||||
that required group. Different list items are ANDed by
|
||||
``missing_required_terms``.
|
||||
"""
|
||||
return [
|
||||
_normalize_match_text(part)
|
||||
for part in str(term_group or "").split("|")
|
||||
if _normalize_match_text(part)
|
||||
]
|
||||
|
||||
|
||||
def missing_required_terms(
|
||||
candidate: AssetCandidate,
|
||||
required_terms: tuple[str, ...] | list[str] | None,
|
||||
) -> list[str]:
|
||||
"""Return required term groups not present in candidate metadata.
|
||||
|
||||
This is an entity-safety gate, not a fuzzy visual classifier. Use it for
|
||||
exact subjects such as landmarks, people, companies, or products where a
|
||||
visually nice but wrong image is worse than no image.
|
||||
"""
|
||||
if not required_terms:
|
||||
return []
|
||||
|
||||
text = _normalize_match_text(_candidate_text(candidate))
|
||||
compact_text = text.replace(" ", "")
|
||||
missing: list[str] = []
|
||||
for group in required_terms:
|
||||
alternatives = _term_group_alternatives(group)
|
||||
if not alternatives:
|
||||
continue
|
||||
matched = any(
|
||||
alt in text or alt.replace(" ", "") in compact_text
|
||||
for alt in alternatives
|
||||
)
|
||||
if not matched:
|
||||
missing.append(str(group))
|
||||
return missing
|
||||
|
||||
|
||||
def compute_relevance(candidate: AssetCandidate, query: str) -> float:
|
||||
"""Fraction of query tokens that appear in the candidate's metadata.
|
||||
|
||||
Range ``[0.0, 1.0]``. Returns ``1.0`` (neutral) when the query has no
|
||||
ASCII tokens to match — this lets non-English queries fall through
|
||||
to license / size scoring without being unfairly rejected.
|
||||
"""
|
||||
tokens = _query_tokens(query)
|
||||
if not tokens:
|
||||
return 1.0
|
||||
text = _candidate_text(candidate)
|
||||
if not text:
|
||||
return 0.0
|
||||
hits = sum(1 for t in tokens if t in text)
|
||||
return hits / len(tokens)
|
||||
|
||||
|
||||
def score_candidate(candidate: AssetCandidate, request: ImageSearchRequest) -> float:
|
||||
"""Score a candidate against a request. Higher is better; -inf rejects.
|
||||
|
||||
Relevance dominates: a candidate whose metadata shares no query
|
||||
tokens is rejected outright, so size / license / orientation cannot
|
||||
rescue an irrelevant image from a permissive provider.
|
||||
"""
|
||||
if not candidate.license_tier:
|
||||
return float("-inf")
|
||||
if (
|
||||
candidate.license_tier == LICENSE_TIER_ATTRIBUTION_REQUIRED
|
||||
and not candidate.author.strip()
|
||||
):
|
||||
return float("-inf")
|
||||
|
||||
required_misses = missing_required_terms(candidate, request.required_terms)
|
||||
if required_misses:
|
||||
return float("-inf")
|
||||
|
||||
relevance = compute_relevance(candidate, request.query)
|
||||
if relevance == 0.0 and not request.required_terms:
|
||||
return float("-inf")
|
||||
|
||||
score = relevance * 10000.0
|
||||
title_text = _normalize_match_text(candidate.title)
|
||||
compact_title = title_text.replace(" ", "")
|
||||
for group in request.required_terms or ():
|
||||
alternatives = _term_group_alternatives(group)
|
||||
if any(
|
||||
alt in title_text or alt.replace(" ", "") in compact_title
|
||||
for alt in alternatives
|
||||
):
|
||||
score += 1500.0
|
||||
|
||||
# Penalize infrastructure/transit metadata if the user didn't explicitly ask for it.
|
||||
# This prevents high-res subway station photos from outranking actual tourist landmarks.
|
||||
text = _candidate_text(candidate)
|
||||
query_lower = request.query.lower()
|
||||
infra_terms = [
|
||||
"station", "subway", "metro", "rail", "transit", "airport", "bus",
|
||||
"地铁", "站", "轨道",
|
||||
]
|
||||
|
||||
if not any(t in query_lower for t in infra_terms):
|
||||
if any(t in text for t in infra_terms):
|
||||
score -= 5000.0
|
||||
|
||||
candidate_orientation = normalize_orientation(candidate.width, candidate.height)
|
||||
requested = (request.orientation or "").strip().lower()
|
||||
if requested:
|
||||
if candidate_orientation == requested:
|
||||
score += 1000.0
|
||||
else:
|
||||
score -= 250.0
|
||||
|
||||
if request.min_width and candidate.width < request.min_width:
|
||||
score -= 500.0
|
||||
if request.min_height and candidate.height < request.min_height:
|
||||
score -= 500.0
|
||||
|
||||
if candidate.license_tier == LICENSE_TIER_NO_ATTRIBUTION:
|
||||
score += 250.0
|
||||
|
||||
# Larger images score higher, but only as a tie-breaker; entity accuracy
|
||||
# and metadata relevance must dominate pixel count.
|
||||
pixel_score = max(candidate.width, 0) * max(candidate.height, 0) / 1000.0
|
||||
score += min(pixel_score, 1500.0)
|
||||
return score
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Attribution text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
PROVIDER_DISPLAY_NAMES: dict[str, str] = {
|
||||
"openverse": "Openverse",
|
||||
"wikimedia": "Wikimedia Commons",
|
||||
"pexels": "Pexels",
|
||||
"pixabay": "Pixabay",
|
||||
}
|
||||
|
||||
|
||||
def build_attribution_text(filename: str, candidate: AssetCandidate) -> str:
|
||||
"""Render the canonical attribution string for the manifest.
|
||||
|
||||
Format:
|
||||
``filename — "title" by author, via Provider, license: name (url)``
|
||||
|
||||
Empty fields are gracefully omitted. The text is intended for use by
|
||||
the Executor when generating in-SVG credit elements; it is not meant
|
||||
to be machine-parsed downstream.
|
||||
"""
|
||||
provider_name = PROVIDER_DISPLAY_NAMES.get(
|
||||
candidate.provider, candidate.provider or "unknown"
|
||||
)
|
||||
|
||||
parts: list[str] = [filename or candidate.download_url or "image"]
|
||||
middle: list[str] = []
|
||||
if candidate.title:
|
||||
middle.append(f'"{candidate.title}"')
|
||||
if candidate.author:
|
||||
middle.append(f"by {candidate.author}")
|
||||
middle.append(f"via {provider_name}")
|
||||
parts.append(" ".join(middle))
|
||||
|
||||
license_part = candidate.license_name or candidate.license_url
|
||||
if license_part:
|
||||
if candidate.license_url and candidate.license_name:
|
||||
license_part = f"{candidate.license_name} ({candidate.license_url})"
|
||||
parts.append(f"license: {license_part}")
|
||||
|
||||
return " — ".join(parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Small helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ensure_json_parent(path: str | Path) -> Path:
|
||||
"""Make sure the parent directory of ``path`` exists; return as Path."""
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
Reference in New Issue
Block a user