Release standalone PPT template workflow 0.5.0
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
"""Web image source providers for PPT Master.
|
||||
|
||||
Each ``provider_<name>.py`` module exposes a ``search(request, *,
|
||||
license_tier_filter)`` function returning a list of ``AssetCandidate``
|
||||
objects, plus a ``parse_results(payload)`` helper. ``image_search.py``
|
||||
dispatches to them by name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,564 @@
|
||||
"""Shared primitives for web image providers.
|
||||
|
||||
This module is the single home for everything that all four providers
|
||||
(Openverse / Wikimedia / Pexels / Pixabay) need:
|
||||
|
||||
- License tier classification (the central abstraction of this module)
|
||||
- Search request / asset candidate dataclasses
|
||||
- Query simplification for keyword-based image APIs
|
||||
- Candidate scoring
|
||||
- Attribution text builder
|
||||
- Small helpers (orientation, json path, etc.)
|
||||
|
||||
Provider-specific code (API URLs, payload shape, parse_results) lives in
|
||||
the corresponding provider_<name>.py module and only imports from here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("This is an internal helper module used by image_search.py and the four web image providers.")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project-wide constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
USER_AGENT = "PPTMaster/1.0 (https://github.com/hugohe3/ppt-master)"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# License tier classification
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Every accepted candidate is classified into exactly one of two tiers:
|
||||
#
|
||||
# "no-attribution" -> No on-slide credit needed (CC0, PD, Pexels,
|
||||
# Pixabay). Default search target.
|
||||
# "attribution-required" -> CC BY / CC BY-SA. Executor must add an
|
||||
# inline credit text element on the slide.
|
||||
#
|
||||
# Anything else (CC BY-NC, CC BY-ND, all-rights-reserved, unknown) returns
|
||||
# None and the candidate is rejected outright.
|
||||
|
||||
LICENSE_TIER_NO_ATTRIBUTION = "no-attribution"
|
||||
LICENSE_TIER_ATTRIBUTION_REQUIRED = "attribution-required"
|
||||
|
||||
# Tokens that mark a license as "no attribution required".
|
||||
NO_ATTRIBUTION_TOKENS: tuple[str, ...] = (
|
||||
"cc0",
|
||||
"public domain",
|
||||
"publicdomain",
|
||||
"creativecommons.org/publicdomain/",
|
||||
"pexels license",
|
||||
"pixabay content license",
|
||||
"pixabay license",
|
||||
)
|
||||
|
||||
# Tokens that mark a license as "attribution required".
|
||||
ATTRIBUTION_REQUIRED_TOKENS: tuple[str, ...] = (
|
||||
"cc by",
|
||||
"cc-by",
|
||||
"by-sa",
|
||||
"by sa",
|
||||
"creativecommons.org/licenses/by/",
|
||||
"creativecommons.org/licenses/by-sa/",
|
||||
)
|
||||
|
||||
# Tokens that disqualify a candidate entirely.
|
||||
REJECTED_TOKENS: tuple[str, ...] = (
|
||||
"by-nc",
|
||||
"by nc",
|
||||
"noncommercial",
|
||||
"non-commercial",
|
||||
"by-nd",
|
||||
"by nd",
|
||||
"no derivatives",
|
||||
"noderivatives",
|
||||
"all rights reserved",
|
||||
)
|
||||
|
||||
|
||||
# Canonical display forms for license names. Different providers report
|
||||
# the same license with different capitalization (Openverse: "cc0",
|
||||
# Wikimedia: "Public domain"); the Executor renders these as on-slide
|
||||
# text, so a normalized form prevents inconsistent credits.
|
||||
_LICENSE_NAME_CANON: dict[str, str] = {
|
||||
"cc0": "CC0",
|
||||
"cc 0": "CC0",
|
||||
"public domain": "Public Domain",
|
||||
"publicdomain": "Public Domain",
|
||||
"pdm": "Public Domain",
|
||||
"pexels license": "Pexels License",
|
||||
"pixabay content license": "Pixabay Content License",
|
||||
"pixabay license": "Pixabay Content License",
|
||||
}
|
||||
|
||||
# CC license short-name pattern used to canonicalize "cc by 4.0" → "CC BY 4.0".
|
||||
_CC_PATTERN = re.compile(
|
||||
r"^\s*cc[\s-]+(by(?:[\s-]+(?:sa|nc|nd))*)\s*([0-9.]*)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_license_name(name: str) -> str:
|
||||
"""Return a canonical display form for a license name.
|
||||
|
||||
Maps common aliases to a consistent capitalization so the on-slide
|
||||
credit text written by the Executor is uniform across providers.
|
||||
Unknown inputs are returned trimmed but otherwise unchanged.
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
key = name.strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
|
||||
if key in _LICENSE_NAME_CANON:
|
||||
return _LICENSE_NAME_CANON[key]
|
||||
|
||||
cc_match = _CC_PATTERN.match(key)
|
||||
if cc_match:
|
||||
suffix_raw, version = cc_match.group(1), cc_match.group(2)
|
||||
suffix = suffix_raw.replace(" ", "-").upper()
|
||||
return f"CC {suffix} {version}".strip()
|
||||
|
||||
return name.strip()
|
||||
|
||||
|
||||
def classify_license(
|
||||
license_name: str,
|
||||
license_url: str = "",
|
||||
provider: str = "",
|
||||
) -> Optional[str]:
|
||||
"""Classify a license string into one of the two tiers, or reject it.
|
||||
|
||||
Returns:
|
||||
``"no-attribution"`` / ``"attribution-required"`` / ``None``.
|
||||
|
||||
The provider hint lets us treat Pexels and Pixabay's own licenses as
|
||||
``no-attribution`` even when the upstream API only returns a short
|
||||
label like ``"Pexels"``.
|
||||
"""
|
||||
text = " ".join(
|
||||
part.strip().lower()
|
||||
for part in (license_name or "", license_url or "")
|
||||
if part
|
||||
)
|
||||
provider_key = (provider or "").strip().lower()
|
||||
|
||||
if not text and not provider_key:
|
||||
return None
|
||||
|
||||
if any(token in text for token in REJECTED_TOKENS):
|
||||
return None
|
||||
|
||||
if any(token in text for token in NO_ATTRIBUTION_TOKENS):
|
||||
return LICENSE_TIER_NO_ATTRIBUTION
|
||||
|
||||
# Provider-default fallback: pexels / pixabay items often arrive with a
|
||||
# bare "Pexels" / "Pixabay" license string. Their site-wide license is
|
||||
# "free for commercial use, no attribution required".
|
||||
#
|
||||
# Guard: require the license text to actually mention the provider name,
|
||||
# so an empty / missing license field never silently passes as no-attribution.
|
||||
if (
|
||||
provider_key in {"pexels", "pixabay"}
|
||||
and provider_key in text
|
||||
and not any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS)
|
||||
):
|
||||
return LICENSE_TIER_NO_ATTRIBUTION
|
||||
|
||||
if any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS):
|
||||
return LICENSE_TIER_ATTRIBUTION_REQUIRED
|
||||
|
||||
return None # unknown license -> reject
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageSearchRequest:
|
||||
"""A single image search intent passed to a provider."""
|
||||
|
||||
query: str
|
||||
purpose: str = ""
|
||||
orientation: str = "" # "landscape" / "portrait" / "square" / ""
|
||||
min_width: int = 0
|
||||
min_height: int = 0
|
||||
filename: str = ""
|
||||
slide: str = ""
|
||||
required_terms: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetCandidate:
|
||||
"""One ranked candidate returned by a provider's parse_results."""
|
||||
|
||||
provider: str
|
||||
title: str
|
||||
asset_id: str = ""
|
||||
source_page_url: str = ""
|
||||
license_name: str = ""
|
||||
license_url: str = ""
|
||||
license_tier: str = "" # one of LICENSE_TIER_* constants
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
download_url: str = ""
|
||||
author: str = ""
|
||||
raw: Any = field(default=None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query simplification
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Web image APIs do keyword matching against image metadata, not semantic
|
||||
# search. Long, descriptive queries with brand names, HEX codes, and
|
||||
# composition notes return zero results. We progressively trim the query
|
||||
# down to the most concrete nouns.
|
||||
|
||||
_NOISE_WORDS = frozenset({
|
||||
# Brand / product names
|
||||
"claude", "openai", "gpt", "gemini", "copilot", "chatgpt", "midjourney",
|
||||
"stable", "diffusion", "dall-e", "cursor", "anthropic", "microsoft",
|
||||
"google", "apple", "meta", "nvidia", "tesla",
|
||||
# Generic filler
|
||||
"using", "with", "from", "that", "this", "have", "been", "will",
|
||||
"into", "more", "also", "very", "some", "than", "them", "other",
|
||||
})
|
||||
|
||||
# Words that look generic but are actually useful when they ARE the
|
||||
# subject of the deck (e.g. a deck about AI). We only drop them when
|
||||
# there are still other concrete nouns left.
|
||||
_SOFT_NOISE_WORDS = frozenset({
|
||||
"ai", "code", "software", "system", "digital", "platform", "solution",
|
||||
"application", "interface", "framework", "algorithm", "api", "sdk",
|
||||
"assistant", "tool", "service", "technology", "tech", "program",
|
||||
# Visual-quality / usage terms. These are helpful in the full provider
|
||||
# query, but should not consume the 3-4 keyword fallback budget or
|
||||
# dominate relevance scoring over the real subject.
|
||||
"professional", "editorial", "commercial", "premium", "stock",
|
||||
"photo", "photograph", "photography", "image", "picture", "visual",
|
||||
"background", "hero", "cover", "banner", "wallpaper",
|
||||
"high", "quality", "resolution", "sharp", "clean", "cinematic",
|
||||
"dramatic", "lighting", "light", "modern", "natural", "visible",
|
||||
})
|
||||
|
||||
_TOKEN_STRIP_CHARS = ".,;:!?\"'()[]{}，。；：！？、"
|
||||
_MATCH_SEPARATOR_RE = re.compile(r"""[\s\-_./:;,'"()[\]{}]+""")
|
||||
|
||||
|
||||
def simplify_query(query: str, max_words: int = 4) -> str:
|
||||
"""Trim a verbose query into a short keyword phrase.
|
||||
|
||||
Strategy:
|
||||
1. Strip HEX color codes and parenthetical asides.
|
||||
2. Drop hard-noise words (brand names, generic filler).
|
||||
3. Drop soft-noise words ONLY if concrete nouns remain.
|
||||
4. If the result would be empty, return the original query
|
||||
(fail-open: better an over-broad search than zero results).
|
||||
5. Cap at ``max_words`` words.
|
||||
"""
|
||||
cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query)
|
||||
cleaned = re.sub(r"\([^)]*\)", "", cleaned)
|
||||
words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
|
||||
words = [w for w in words if len(w) > 2]
|
||||
|
||||
after_hard = [w for w in words if w.lower() not in _NOISE_WORDS]
|
||||
after_soft = [w for w in after_hard if w.lower() not in _SOFT_NOISE_WORDS]
|
||||
|
||||
# Only drop soft-noise if there are still concrete nouns left.
|
||||
filtered = after_soft if after_soft else after_hard
|
||||
|
||||
if not filtered:
|
||||
# Everything got filtered. Fail open: return the original query.
|
||||
return query.strip()
|
||||
|
||||
return " ".join(filtered[:max_words])
|
||||
|
||||
|
||||
def build_query_progression(query: str) -> list[str]:
|
||||
"""Return a list of progressively simpler queries to try in order.
|
||||
|
||||
Stops as soon as one of them yields candidates upstream. Duplicates
|
||||
are dropped while preserving order.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for candidate in (
|
||||
query,
|
||||
simplify_query(query, max_words=4),
|
||||
simplify_query(query, max_words=3),
|
||||
simplify_query(query, max_words=2),
|
||||
simplify_query(query, max_words=1),
|
||||
):
|
||||
candidate = candidate.strip()
|
||||
if candidate and candidate not in seen:
|
||||
seen.add(candidate)
|
||||
out.append(candidate)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normalize_orientation(width: int, height: int) -> str:
|
||||
if width <= 0 or height <= 0:
|
||||
return "unknown"
|
||||
if width > height:
|
||||
return "landscape"
|
||||
if height > width:
|
||||
return "portrait"
|
||||
return "square"
|
||||
|
||||
|
||||
def _query_tokens(query: str) -> list[str]:
|
||||
"""Extract ASCII keyword tokens from a query for relevance scoring.
|
||||
|
||||
Uses the same noise-word filtering as ``simplify_query`` so the
|
||||
relevance signal lines up with the keywords we actually search by.
|
||||
Non-ASCII tokens (CJK etc.) are dropped — image metadata is mostly
|
||||
English even on multi-language providers, so substring matching CJK
|
||||
against an English title is unreliable. When this leaves no tokens,
|
||||
``compute_relevance`` falls back to neutral (1.0) and lets the other
|
||||
score dimensions decide.
|
||||
"""
|
||||
cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query.lower())
|
||||
cleaned = re.sub(r"\([^)]*\)", "", cleaned)
|
||||
words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
|
||||
words = [w for w in words if len(w) > 2 and w.isascii()]
|
||||
if not words:
|
||||
return []
|
||||
after_hard = [w for w in words if w not in _NOISE_WORDS]
|
||||
after_soft = [w for w in after_hard if w not in _SOFT_NOISE_WORDS]
|
||||
return after_soft if after_soft else after_hard
|
||||
|
||||
|
||||
def _candidate_text(candidate: AssetCandidate) -> str:
|
||||
"""Concatenate the candidate's matchable metadata fields for scoring."""
|
||||
return " ".join(
|
||||
filter(
|
||||
None,
|
||||
(
|
||||
candidate.title,
|
||||
candidate.author,
|
||||
candidate.source_page_url,
|
||||
),
|
||||
)
|
||||
).lower()
|
||||
|
||||
|
||||
def _normalize_match_text(text: str) -> str:
|
||||
"""Normalize metadata / required terms for conservative substring matching."""
|
||||
lowered = (text or "").lower()
|
||||
return _MATCH_SEPARATOR_RE.sub(" ", lowered).strip()
|
||||
|
||||
|
||||
def _term_group_alternatives(term_group: str) -> list[str]:
|
||||
"""Split one required term group into alternatives.
|
||||
|
||||
``"Jiefangbei|Liberation Monument"`` means either alternative satisfies
|
||||
that required group. Different list items are ANDed by
|
||||
``missing_required_terms``.
|
||||
"""
|
||||
return [
|
||||
_normalize_match_text(part)
|
||||
for part in str(term_group or "").split("|")
|
||||
if _normalize_match_text(part)
|
||||
]
|
||||
|
||||
|
||||
def missing_required_terms(
|
||||
candidate: AssetCandidate,
|
||||
required_terms: tuple[str, ...] | list[str] | None,
|
||||
) -> list[str]:
|
||||
"""Return required term groups not present in candidate metadata.
|
||||
|
||||
This is an entity-safety gate, not a fuzzy visual classifier. Use it for
|
||||
exact subjects such as landmarks, people, companies, or products where a
|
||||
visually nice but wrong image is worse than no image.
|
||||
"""
|
||||
if not required_terms:
|
||||
return []
|
||||
|
||||
text = _normalize_match_text(_candidate_text(candidate))
|
||||
compact_text = text.replace(" ", "")
|
||||
missing: list[str] = []
|
||||
for group in required_terms:
|
||||
alternatives = _term_group_alternatives(group)
|
||||
if not alternatives:
|
||||
continue
|
||||
matched = any(
|
||||
alt in text or alt.replace(" ", "") in compact_text
|
||||
for alt in alternatives
|
||||
)
|
||||
if not matched:
|
||||
missing.append(str(group))
|
||||
return missing
|
||||
|
||||
|
||||
def compute_relevance(candidate: AssetCandidate, query: str) -> float:
|
||||
"""Fraction of query tokens that appear in the candidate's metadata.
|
||||
|
||||
Range ``[0.0, 1.0]``. Returns ``1.0`` (neutral) when the query has no
|
||||
ASCII tokens to match — this lets non-English queries fall through
|
||||
to license / size scoring without being unfairly rejected.
|
||||
"""
|
||||
tokens = _query_tokens(query)
|
||||
if not tokens:
|
||||
return 1.0
|
||||
text = _candidate_text(candidate)
|
||||
if not text:
|
||||
return 0.0
|
||||
hits = sum(1 for t in tokens if t in text)
|
||||
return hits / len(tokens)
|
||||
|
||||
|
||||
def score_candidate(candidate: AssetCandidate, request: ImageSearchRequest) -> float:
|
||||
"""Score a candidate against a request. Higher is better; -inf rejects.
|
||||
|
||||
Relevance dominates: a candidate whose metadata shares no query
|
||||
tokens is rejected outright, so size / license / orientation cannot
|
||||
rescue an irrelevant image from a permissive provider.
|
||||
"""
|
||||
if not candidate.license_tier:
|
||||
return float("-inf")
|
||||
if (
|
||||
candidate.license_tier == LICENSE_TIER_ATTRIBUTION_REQUIRED
|
||||
and not candidate.author.strip()
|
||||
):
|
||||
return float("-inf")
|
||||
|
||||
required_misses = missing_required_terms(candidate, request.required_terms)
|
||||
if required_misses:
|
||||
return float("-inf")
|
||||
|
||||
relevance = compute_relevance(candidate, request.query)
|
||||
if relevance == 0.0 and not request.required_terms:
|
||||
return float("-inf")
|
||||
|
||||
score = relevance * 10000.0
|
||||
title_text = _normalize_match_text(candidate.title)
|
||||
compact_title = title_text.replace(" ", "")
|
||||
for group in request.required_terms or ():
|
||||
alternatives = _term_group_alternatives(group)
|
||||
if any(
|
||||
alt in title_text or alt.replace(" ", "") in compact_title
|
||||
for alt in alternatives
|
||||
):
|
||||
score += 1500.0
|
||||
|
||||
# Penalize infrastructure/transit metadata if the user didn't explicitly ask for it.
|
||||
# This prevents high-res subway station photos from outranking actual tourist landmarks.
|
||||
text = _candidate_text(candidate)
|
||||
query_lower = request.query.lower()
|
||||
infra_terms = [
|
||||
"station", "subway", "metro", "rail", "transit", "airport", "bus",
|
||||
"地铁", "站", "轨道",
|
||||
]
|
||||
|
||||
if not any(t in query_lower for t in infra_terms):
|
||||
if any(t in text for t in infra_terms):
|
||||
score -= 5000.0
|
||||
|
||||
candidate_orientation = normalize_orientation(candidate.width, candidate.height)
|
||||
requested = (request.orientation or "").strip().lower()
|
||||
if requested:
|
||||
if candidate_orientation == requested:
|
||||
score += 1000.0
|
||||
else:
|
||||
score -= 250.0
|
||||
|
||||
if request.min_width and candidate.width < request.min_width:
|
||||
score -= 500.0
|
||||
if request.min_height and candidate.height < request.min_height:
|
||||
score -= 500.0
|
||||
|
||||
if candidate.license_tier == LICENSE_TIER_NO_ATTRIBUTION:
|
||||
score += 250.0
|
||||
|
||||
# Larger images score higher, but only as a tie-breaker; entity accuracy
|
||||
# and metadata relevance must dominate pixel count.
|
||||
pixel_score = max(candidate.width, 0) * max(candidate.height, 0) / 1000.0
|
||||
score += min(pixel_score, 1500.0)
|
||||
return score
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Attribution text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
PROVIDER_DISPLAY_NAMES: dict[str, str] = {
|
||||
"openverse": "Openverse",
|
||||
"wikimedia": "Wikimedia Commons",
|
||||
"pexels": "Pexels",
|
||||
"pixabay": "Pixabay",
|
||||
}
|
||||
|
||||
|
||||
def build_attribution_text(filename: str, candidate: AssetCandidate) -> str:
|
||||
"""Render the canonical attribution string for the manifest.
|
||||
|
||||
Format:
|
||||
``filename — "title" by author, via Provider, license: name (url)``
|
||||
|
||||
Empty fields are gracefully omitted. The text is intended for use by
|
||||
the Executor when generating in-SVG credit elements; it is not meant
|
||||
to be machine-parsed downstream.
|
||||
"""
|
||||
provider_name = PROVIDER_DISPLAY_NAMES.get(
|
||||
candidate.provider, candidate.provider or "unknown"
|
||||
)
|
||||
|
||||
parts: list[str] = [filename or candidate.download_url or "image"]
|
||||
middle: list[str] = []
|
||||
if candidate.title:
|
||||
middle.append(f'"{candidate.title}"')
|
||||
if candidate.author:
|
||||
middle.append(f"by {candidate.author}")
|
||||
middle.append(f"via {provider_name}")
|
||||
parts.append(" ".join(middle))
|
||||
|
||||
license_part = candidate.license_name or candidate.license_url
|
||||
if license_part:
|
||||
if candidate.license_url and candidate.license_name:
|
||||
license_part = f"{candidate.license_name} ({candidate.license_url})"
|
||||
parts.append(f"license: {license_part}")
|
||||
|
||||
return " — ".join(parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Small helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ensure_json_parent(path: str | Path) -> Path:
|
||||
"""Make sure the parent directory of ``path`` exists; return as Path."""
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Openverse provider.
|
||||
|
||||
Zero-config (no API key required). Indexes openly licensed images across
|
||||
Wikimedia, Flickr, museums, and other sources.
|
||||
|
||||
API docs: https://api.openverse.org/v1/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("Use via: python3 skills/ppt-master/scripts/image_search.py ...")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import requests
|
||||
|
||||
from image_sources.provider_common import (
|
||||
AssetCandidate,
|
||||
ImageSearchRequest,
|
||||
USER_AGENT,
|
||||
build_query_progression,
|
||||
classify_license,
|
||||
normalize_license_name,
|
||||
)
|
||||
|
||||
|
||||
API_URL = "https://api.openverse.org/v1/images/"
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
# Map our orientation vocabulary to Openverse's ``aspect_ratio`` parameter.
|
||||
_ASPECT_MAP = {"landscape": "wide", "portrait": "tall", "square": "square"}
|
||||
|
||||
# Openverse license param values. ``cc0,pdm`` covers our "no-attribution" tier;
|
||||
# adding ``by,by-sa`` opens the "attribution-required" tier.
|
||||
_LICENSE_PARAM = {
|
||||
"no-attribution-only": "cc0,pdm",
|
||||
"all": "by,by-sa,cc0,pdm",
|
||||
}
|
||||
|
||||
|
||||
def parse_results(payload: dict) -> list[AssetCandidate]:
|
||||
"""Translate an Openverse search payload into a list of candidates."""
|
||||
candidates: list[AssetCandidate] = []
|
||||
for item in payload.get("results", []) or []:
|
||||
license_name = (item.get("license") or "").strip()
|
||||
license_url = (item.get("license_url") or "").strip()
|
||||
tier = classify_license(license_name, license_url, provider="openverse")
|
||||
if not tier:
|
||||
continue
|
||||
|
||||
download_url = (item.get("url") or item.get("thumbnail") or "").strip()
|
||||
if not download_url:
|
||||
continue
|
||||
|
||||
candidates.append(
|
||||
AssetCandidate(
|
||||
provider="openverse",
|
||||
title=(item.get("title") or "").strip() or "Untitled",
|
||||
asset_id=str(item.get("id") or ""),
|
||||
source_page_url=(
|
||||
item.get("foreign_landing_url") or item.get("detail_url") or ""
|
||||
).strip(),
|
||||
license_name=normalize_license_name(license_name),
|
||||
license_url=license_url,
|
||||
license_tier=tier,
|
||||
width=int(item.get("width") or 0),
|
||||
height=int(item.get("height") or 0),
|
||||
download_url=download_url,
|
||||
author=(item.get("creator") or "").strip(),
|
||||
raw=item,
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def search(
|
||||
request: ImageSearchRequest,
|
||||
*,
|
||||
license_tier_filter: str = "no-attribution-only",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
) -> list[AssetCandidate]:
|
||||
"""Search Openverse for candidates matching ``request``.
|
||||
|
||||
``license_tier_filter`` is one of ``"no-attribution-only"`` or ``"all"``.
|
||||
Returns the candidates from the first non-empty query in the
|
||||
progression — caller is responsible for picking the best one via
|
||||
``score_candidate``.
|
||||
"""
|
||||
if license_tier_filter not in _LICENSE_PARAM:
|
||||
raise ValueError(f"unsupported license_tier_filter: {license_tier_filter!r}")
|
||||
|
||||
orientation = (request.orientation or "").strip().lower()
|
||||
|
||||
for query in build_query_progression(request.query):
|
||||
params: dict[str, str | int] = {
|
||||
"q": query,
|
||||
"page_size": page_size,
|
||||
"license": _LICENSE_PARAM[license_tier_filter],
|
||||
"size": "large",
|
||||
}
|
||||
if orientation in _ASPECT_MAP:
|
||||
params["aspect_ratio"] = _ASPECT_MAP[orientation]
|
||||
|
||||
response = requests.get(
|
||||
API_URL,
|
||||
params=params,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
candidates = parse_results(response.json())
|
||||
if candidates:
|
||||
return candidates
|
||||
|
||||
return []
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Pexels provider.
|
||||
|
||||
Requires ``PEXELS_API_KEY`` in the environment. Pexels's site-wide license
|
||||
allows commercial use without attribution, so all returned candidates are
|
||||
classified as ``no-attribution``.
|
||||
|
||||
API docs: https://www.pexels.com/api/documentation/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("Use via: python3 skills/ppt-master/scripts/image_search.py ...")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
from image_sources.provider_common import (
|
||||
AssetCandidate,
|
||||
ImageSearchRequest,
|
||||
LICENSE_TIER_NO_ATTRIBUTION,
|
||||
USER_AGENT,
|
||||
build_query_progression,
|
||||
normalize_license_name,
|
||||
)
|
||||
|
||||
|
||||
API_URL = "https://api.pexels.com/v1/search"
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
_ORIENTATION_MAP = {
|
||||
"landscape": "landscape",
|
||||
"portrait": "portrait",
|
||||
"square": "square",
|
||||
}
|
||||
|
||||
|
||||
def _require_api_key() -> str:
|
||||
key = (os.environ.get("PEXELS_API_KEY") or "").strip()
|
||||
if not key:
|
||||
raise RuntimeError(
|
||||
"PEXELS_API_KEY is not set. Add it to your environment or .env file. "
|
||||
"Get one at https://www.pexels.com/api/"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def parse_results(payload: dict) -> list[AssetCandidate]:
|
||||
"""Translate a Pexels response into candidates."""
|
||||
candidates: list[AssetCandidate] = []
|
||||
for item in payload.get("photos", []) or []:
|
||||
src = item.get("src") or {}
|
||||
download_url = (src.get("original") or src.get("large2x") or src.get("large") or "").strip()
|
||||
if not download_url:
|
||||
continue
|
||||
|
||||
candidates.append(
|
||||
AssetCandidate(
|
||||
provider="pexels",
|
||||
title=(item.get("alt") or "").strip() or "Pexels photo",
|
||||
asset_id=str(item.get("id") or ""),
|
||||
source_page_url=(item.get("url") or "").strip(),
|
||||
license_name=normalize_license_name("Pexels License"),
|
||||
license_url="https://www.pexels.com/license/",
|
||||
license_tier=LICENSE_TIER_NO_ATTRIBUTION,
|
||||
width=int(item.get("width") or 0),
|
||||
height=int(item.get("height") or 0),
|
||||
download_url=download_url,
|
||||
author=(item.get("photographer") or "").strip(),
|
||||
raw=item,
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def search(
|
||||
request: ImageSearchRequest,
|
||||
*,
|
||||
license_tier_filter: str = "no-attribution-only",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
) -> list[AssetCandidate]:
|
||||
"""Search Pexels for candidates.
|
||||
|
||||
Pexels images are uniformly ``no-attribution``, so the ``"all"`` filter
|
||||
behaves identically to ``"no-attribution-only"``. Both are accepted to
|
||||
keep the dispatcher simple.
|
||||
"""
|
||||
if license_tier_filter not in {"no-attribution-only", "all"}:
|
||||
raise ValueError(f"unsupported license_tier_filter: {license_tier_filter!r}")
|
||||
|
||||
api_key = _require_api_key()
|
||||
orientation = (request.orientation or "").strip().lower()
|
||||
|
||||
for query in build_query_progression(request.query):
|
||||
params: dict[str, str | int] = {
|
||||
"query": query,
|
||||
"per_page": page_size,
|
||||
"size": "large",
|
||||
}
|
||||
if orientation in _ORIENTATION_MAP:
|
||||
params["orientation"] = _ORIENTATION_MAP[orientation]
|
||||
|
||||
response = requests.get(
|
||||
API_URL,
|
||||
params=params,
|
||||
headers={
|
||||
"Authorization": api_key,
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
candidates = parse_results(response.json())
|
||||
if candidates:
|
||||
return candidates
|
||||
|
||||
return []
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Pixabay provider.
|
||||
|
||||
Requires ``PIXABAY_API_KEY`` in the environment. Pixabay's Content License
|
||||
allows commercial use without attribution, so all returned candidates are
|
||||
classified as ``no-attribution``.
|
||||
|
||||
API docs: https://pixabay.com/api/docs/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("Use via: python3 skills/ppt-master/scripts/image_search.py ...")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
from image_sources.provider_common import (
|
||||
AssetCandidate,
|
||||
ImageSearchRequest,
|
||||
LICENSE_TIER_NO_ATTRIBUTION,
|
||||
USER_AGENT,
|
||||
build_query_progression,
|
||||
normalize_license_name,
|
||||
)
|
||||
|
||||
|
||||
API_URL = "https://pixabay.com/api/"
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
# Pixabay uses ``horizontal`` / ``vertical`` rather than landscape/portrait,
|
||||
# and has no ``square`` value (it falls back to ``all``).
|
||||
_ORIENTATION_MAP = {
|
||||
"landscape": "horizontal",
|
||||
"portrait": "vertical",
|
||||
}
|
||||
|
||||
|
||||
def _require_api_key() -> str:
|
||||
key = (os.environ.get("PIXABAY_API_KEY") or "").strip()
|
||||
if not key:
|
||||
raise RuntimeError(
|
||||
"PIXABAY_API_KEY is not set. Add it to your environment or .env file. "
|
||||
"Get one at https://pixabay.com/api/docs/"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def parse_results(payload: dict) -> list[AssetCandidate]:
|
||||
"""Translate a Pixabay response into candidates."""
|
||||
candidates: list[AssetCandidate] = []
|
||||
for item in payload.get("hits", []) or []:
|
||||
download_url = (
|
||||
item.get("largeImageURL")
|
||||
or item.get("webformatURL")
|
||||
or item.get("previewURL")
|
||||
or ""
|
||||
).strip()
|
||||
if not download_url:
|
||||
continue
|
||||
|
||||
candidates.append(
|
||||
AssetCandidate(
|
||||
provider="pixabay",
|
||||
title=(item.get("tags") or "").strip() or "Pixabay image",
|
||||
asset_id=str(item.get("id") or ""),
|
||||
source_page_url=(item.get("pageURL") or "").strip(),
|
||||
license_name=normalize_license_name("Pixabay Content License"),
|
||||
license_url="https://pixabay.com/service/license-summary/",
|
||||
license_tier=LICENSE_TIER_NO_ATTRIBUTION,
|
||||
width=int(item.get("imageWidth") or 0),
|
||||
height=int(item.get("imageHeight") or 0),
|
||||
download_url=download_url,
|
||||
author=(item.get("user") or "").strip(),
|
||||
raw=item,
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def search(
|
||||
request: ImageSearchRequest,
|
||||
*,
|
||||
license_tier_filter: str = "no-attribution-only",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
) -> list[AssetCandidate]:
|
||||
"""Search Pixabay for candidates.
|
||||
|
||||
Pixabay images are uniformly ``no-attribution``, so the ``"all"`` filter
|
||||
behaves identically to ``"no-attribution-only"``.
|
||||
"""
|
||||
if license_tier_filter not in {"no-attribution-only", "all"}:
|
||||
raise ValueError(f"unsupported license_tier_filter: {license_tier_filter!r}")
|
||||
|
||||
api_key = _require_api_key()
|
||||
orientation = (request.orientation or "").strip().lower()
|
||||
|
||||
for query in build_query_progression(request.query):
|
||||
params: dict[str, str | int] = {
|
||||
"key": api_key,
|
||||
"q": query,
|
||||
"image_type": "photo",
|
||||
"per_page": page_size,
|
||||
"safesearch": "true",
|
||||
}
|
||||
if orientation in _ORIENTATION_MAP:
|
||||
params["orientation"] = _ORIENTATION_MAP[orientation]
|
||||
|
||||
response = requests.get(
|
||||
API_URL,
|
||||
params=params,
|
||||
headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
candidates = parse_results(response.json())
|
||||
if candidates:
|
||||
return candidates
|
||||
|
||||
return []
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Wikimedia Commons provider.
|
||||
|
||||
Zero-config (no API key required). Strong on educational, scientific,
|
||||
geographic, and historical imagery; weaker on contemporary stock-style
|
||||
photography and people.
|
||||
|
||||
Uses the MediaWiki API's ``generator=search`` mode to combine fulltext
|
||||
search with imageinfo/extmetadata in a single round trip.
|
||||
|
||||
API docs: https://www.mediawiki.org/wiki/API:Search
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(_SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||||
|
||||
from console_encoding import configure_utf8_stdio # noqa: E402
|
||||
|
||||
configure_utf8_stdio()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(__doc__)
|
||||
print("Use via: python3 skills/ppt-master/scripts/image_search.py ...")
|
||||
raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
from image_sources.provider_common import (
|
||||
AssetCandidate,
|
||||
ImageSearchRequest,
|
||||
USER_AGENT,
|
||||
build_query_progression,
|
||||
classify_license,
|
||||
normalize_license_name,
|
||||
normalize_orientation,
|
||||
)
|
||||
|
||||
|
||||
API_URL = "https://commons.wikimedia.org/w/api.php"
|
||||
DEFAULT_SEARCH_LIMIT = 20
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
# File extensions we are willing to embed in a deck. SVG/GIF/audio etc. are
|
||||
# excluded — Wikimedia returns these freely from a generic search.
|
||||
_ACCEPTED_EXTENSIONS = frozenset({".jpg", ".jpeg", ".png", ".webp", ".tiff", ".tif"})
|
||||
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _strip_html(value: str) -> str:
|
||||
"""Wikimedia's extmetadata fields contain inline HTML markup. Flatten it."""
|
||||
if not value:
|
||||
return ""
|
||||
text = html.unescape(str(value))
|
||||
text = _TAG_RE.sub(" ", text)
|
||||
text = _WS_RE.sub(" ", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _ext_value(extmetadata: dict, key: str) -> str:
|
||||
"""Pull ``extmetadata[key].value`` and strip HTML."""
|
||||
entry = extmetadata.get(key) or {}
|
||||
if isinstance(entry, dict):
|
||||
return _strip_html(entry.get("value", ""))
|
||||
return _strip_html(entry)
|
||||
|
||||
|
||||
def _accept_extension(title: str) -> bool:
|
||||
"""Drop non-image files (svg/gif/audio/video) by extension."""
|
||||
lower = (title or "").lower()
|
||||
return any(lower.endswith(ext) for ext in _ACCEPTED_EXTENSIONS)
|
||||
|
||||
|
||||
def _page_label(title: str) -> str:
|
||||
"""``File:Some_image.jpg`` → ``Some_image.jpg``."""
|
||||
clean = _strip_html(title)
|
||||
if clean.lower().startswith("file:"):
|
||||
return clean.split(":", 1)[1].strip()
|
||||
return clean
|
||||
|
||||
|
||||
def parse_results(payload: dict) -> list[AssetCandidate]:
|
||||
"""Translate a MediaWiki ``query`` payload into a list of candidates."""
|
||||
candidates: list[AssetCandidate] = []
|
||||
pages = ((payload.get("query") or {}).get("pages") or {})
|
||||
|
||||
for page in pages.values():
|
||||
title = page.get("title") or ""
|
||||
if not _accept_extension(title):
|
||||
continue
|
||||
|
||||
info_list = page.get("imageinfo") or []
|
||||
if not info_list:
|
||||
continue
|
||||
info = info_list[0]
|
||||
extmetadata = info.get("extmetadata") or {}
|
||||
|
||||
license_name = (
|
||||
_ext_value(extmetadata, "LicenseShortName")
|
||||
or _ext_value(extmetadata, "License")
|
||||
)
|
||||
license_url = _ext_value(extmetadata, "LicenseUrl")
|
||||
tier = classify_license(license_name, license_url, provider="wikimedia")
|
||||
if not tier:
|
||||
continue
|
||||
|
||||
download_url = (info.get("url") or "").strip()
|
||||
if not download_url:
|
||||
continue
|
||||
|
||||
candidates.append(
|
||||
AssetCandidate(
|
||||
provider="wikimedia",
|
||||
title=_page_label(title) or "Untitled",
|
||||
asset_id=str(page.get("pageid") or ""),
|
||||
source_page_url=(info.get("descriptionurl") or "").strip(),
|
||||
license_name=normalize_license_name(license_name),
|
||||
license_url=license_url,
|
||||
license_tier=tier,
|
||||
width=int(info.get("width") or 0),
|
||||
height=int(info.get("height") or 0),
|
||||
download_url=download_url,
|
||||
author=_ext_value(extmetadata, "Artist"),
|
||||
raw=page,
|
||||
)
|
||||
)
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
def _filter_by_orientation(
|
||||
candidates: list[AssetCandidate], orientation: str
|
||||
) -> list[AssetCandidate]:
|
||||
"""Wikimedia API has no orientation parameter; filter client-side."""
|
||||
if not orientation or orientation == "any":
|
||||
return candidates
|
||||
matching = [
|
||||
c for c in candidates if normalize_orientation(c.width, c.height) == orientation
|
||||
]
|
||||
# Fall back to the unfiltered list if orientation pruning leaves nothing —
|
||||
# better an off-orientation match than no image at all.
|
||||
return matching or candidates
|
||||
|
||||
|
||||
def search(
|
||||
request: ImageSearchRequest,
|
||||
*,
|
||||
license_tier_filter: str = "no-attribution-only",
|
||||
search_limit: int = DEFAULT_SEARCH_LIMIT,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
) -> list[AssetCandidate]:
|
||||
"""Search Wikimedia Commons for candidates.
|
||||
|
||||
Wikimedia returns license info via ``extmetadata`` rather than as a
|
||||
request parameter, so the ``license_tier_filter`` does its work in
|
||||
``parse_results`` (and is honored implicitly because tier classification
|
||||
happens there). Returning candidates from the first non-empty query.
|
||||
"""
|
||||
if license_tier_filter not in {"no-attribution-only", "all"}:
|
||||
raise ValueError(f"unsupported license_tier_filter: {license_tier_filter!r}")
|
||||
|
||||
orientation = (request.orientation or "").strip().lower()
|
||||
|
||||
for query in build_query_progression(request.query):
|
||||
params = {
|
||||
"action": "query",
|
||||
"format": "json",
|
||||
"generator": "search",
|
||||
"gsrnamespace": "6", # File: namespace
|
||||
"gsrsearch": f"{query} filetype:bitmap",
|
||||
"gsrlimit": search_limit,
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url|size|extmetadata|mime",
|
||||
"iiextmetadatafilter": (
|
||||
"LicenseShortName|License|LicenseUrl|Artist"
|
||||
),
|
||||
}
|
||||
|
||||
response = requests.get(
|
||||
API_URL,
|
||||
params=params,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
all_candidates = parse_results(response.json())
|
||||
|
||||
if license_tier_filter == "no-attribution-only":
|
||||
all_candidates = [
|
||||
c for c in all_candidates if c.license_tier == "no-attribution"
|
||||
]
|
||||
|
||||
all_candidates = _filter_by_orientation(all_candidates, orientation)
|
||||
if all_candidates:
|
||||
return all_candidates
|
||||
|
||||
return []
|
||||
Reference in New Issue
Block a user