♻️ refactor(config): remove CacheManager, CrawlLimits, BaseConfigModel and... (8b7d7e35) · Commits · Jan Reimes / 3gpp-crawler

src/tdoc_crawler/config/init.py

+5 −159

Original line number	Diff line number	Diff line
		"""Configuration management for file paths and caching behavior."""
		"""Configuration management for 3GPP Crawler."""

		from __future__ import annotations

		import os
		from pathlib import Path
		from typing import Self

		# Import settings and sources modules
		from tdoc_crawler.config.env_vars import (
		DEPRECATED_ENV_VARS,
		ENV_VAR_MAPPINGS,
		ConfigEnvVar,
		log_deprecation_warning,
		)
		from tdoc_crawler.config.env_vars import TOML_PATH_TO_ENV_VAR, ConfigEnvVar
		from tdoc_crawler.config.settings import (
		CrawlConfig,
		CredentialsConfig,
		HttpConfig,
		PathConfig,
		TDocCrawlerConfig,
		ThreeGPPConfig,
		)
		from tdoc_crawler.config.sources import (
		ConfigLoadError,
		@@ -27,27 +18,11 @@ from tdoc_crawler.config.sources import (
		merge_configs,
		)

		# Fallback path if no argument or env var is provided
		DEFAULT_CACHE_DIR = Path.home() / ".3gpp-crawler"
		DEFAULT_DATABASE_FILENAME = "3gpp_crawler.db"
		DEFAULT_HTTP_CACHE_FILENAME = "http-cache.sqlite3"
		DEFAULT_CHECKOUT_DIRNAME = "checkout"
		DEFAULT_MANAGER = "default"

		DEFAULT_AI_CACHE_DIRNAME = "lightrag" # subdirectory under root cache dir for AI-related files

		WORKSPACE_REGISTRY_FILENAME = "workspaces.json"

		__all__ = [
		"DEFAULT_AI_CACHE_DIRNAME",
		"DEFAULT_CACHE_DIR",
		"DEFAULT_CHECKOUT_DIRNAME",
		"DEFAULT_DATABASE_FILENAME",
		"DEFAULT_HTTP_CACHE_FILENAME",
		"DEFAULT_MANAGER",
		"DEPRECATED_ENV_VARS",
		"ENV_VAR_MAPPINGS",
		"CacheManager",
		"TOML_PATH_TO_ENV_VAR",
		"ConfigEnvVar",
		"ConfigLoadError",
		"CrawlConfig",
		@@ -55,137 +30,8 @@ __all__ = [
		"HttpConfig",
		"PathConfig",
		"TDocCrawlerConfig",
		"ThreeGPPConfig",
		"discover_config_files",
		"load_config_file",
		"log_deprecation_warning",
		"merge_configs",
		"register_cache_manager",
		"reset_cache_managers",
		"resolve_cache_manager",
		]

		_cache_managers: dict[str, CacheManager] = {}


		def register_cache_manager(manager: CacheManager, force: bool = False) -> None:
		"""Register a cache manager instance under a given name.

		Args:
		manager: CacheManager instance to register
		force: If True, overwrite existing manager with same name
		"""
		name = manager.name
		if name in _cache_managers and not force:
		raise ValueError(f"Cache manager with name '{name}' is already registered.")
		_cache_managers[name] = manager


		def reset_cache_managers() -> None:
		"""Clear all registered cache managers.

		Primarily useful for testing to ensure clean state between tests.
		"""
		_cache_managers.clear()


		def resolve_cache_manager(name: str \| None = None) -> CacheManager:
		"""Resolve a cache manager by name, or return the default if name is None."""
		name = name or DEFAULT_MANAGER
		manager = _cache_managers.get(name)
		if manager is None:
		raise ValueError(f"No cache manager registered under name '{name}'.")
		return manager


		class CacheManager:
		"""Manages cache directory layout and path resolution.

		Acts as the single source of truth for where files are stored.
		"""

		def __init__(
		self,
		root_path: Path \| None = None,
		ai_cache_dir: Path \| None = None,
		name: str = DEFAULT_MANAGER,
		ensure_paths: bool = True,
		config: TDocCrawlerConfig \| None = None,
		) -> None:
		"""Initialize cache manager.

		Args:
		root_path: Explicit root path. If None, tries TDC_CACHE_DIR env var,
		then falls back to DEFAULT_CACHE_DIR. Ignored if config is provided.
		ai_cache_dir: Explicit AI cache directory path. If None, defaults to root_path/lightrag
		or TDC_AI_STORE_PATH env var if set. Ignored if config is provided.
		name: Optional name to register this manager under. If provided, the manager is registered upon initialization.
		ensure_paths: If True, will create the root directory if it doesn't exist.
		config: Optional TDocCrawlerConfig to use for path resolution. If provided,
		config.path.cache_dir and config.path.ai_cache_dir are used.
		"""
		self.name = name

		if config is not None:
		# Use config for path resolution (preferred)
		self.root = config.path.cache_dir
		self.ai_cache_dir = config.path.ai_cache_dir
		elif root_path:
		self.root = root_path
		else:
		# Fallback to env var (will be removed in future version)
		env_cache_dir = os.getenv("TDC_CACHE_DIR")
		self.root = Path(env_cache_dir) if env_cache_dir else DEFAULT_CACHE_DIR

		if config is None and not ai_cache_dir:
		env_ai_cache_dir = os.getenv("TDC_AI_STORE_PATH")
		if env_ai_cache_dir:
		self.ai_cache_dir = Path(env_ai_cache_dir).resolve()
		else:
		self.ai_cache_dir = (self.root / DEFAULT_AI_CACHE_DIRNAME).resolve()
		elif ai_cache_dir:
		self.ai_cache_dir = ai_cache_dir.resolve()

		if ensure_paths:
		self.ensure_paths()
		self.ensure_ai_paths()

		def register(self, force: bool = True) -> Self:
		"""Register this instance as a cache manager under the given name.

		Args:
		force: If True (default), overwrite existing manager with same name
		"""
		register_cache_manager(self, force=force)
		return self

		@property
		def http_cache_file(self) -> Path:
		"""Path to the HTTP client cache database file."""
		return self.root / DEFAULT_HTTP_CACHE_FILENAME

		@property
		def db_file(self) -> Path:
		"""Path to the metadata SQLite database."""
		return self.root / DEFAULT_DATABASE_FILENAME

		@property
		def checkout_dir(self) -> Path:
		"""Path to the default checkout directory."""
		return self.root / DEFAULT_CHECKOUT_DIRNAME

		def ai_embed_dir(self, embedding_model: str) -> Path:
		"""Path to store AI-related files for a specific embedding model."""
		return (self.ai_cache_dir / embedding_model).resolve()

		@property
		def ai_workspace_file(self) -> Path:
		"""Path to the workspace registry file for a specific workspace."""
		return self.ai_cache_dir / WORKSPACE_REGISTRY_FILENAME

		def ensure_paths(self) -> None:
		"""Ensure the root cache directory exists."""
		self.root.mkdir(parents=True, exist_ok=True)

		def ensure_ai_paths(self) -> None:
		"""Ensure the AI cache directory exists."""
		self.ai_cache_dir.mkdir(parents=True, exist_ok=True)

src/tdoc_crawler/config/env_vars.py

+4 −36

Original line number	Diff line number	Diff line
		"""
		Environment variable name constants mapped to their corresponding config field paths.
		"""Environment variable name constants mapped to their corresponding config field paths.

		All TDC_, HTTP_CACHE_, and LIGHTRAG_* environment variables remain functional
		via pydantic's AliasChoices mechanism.
		@@ -14,17 +13,14 @@ Use these constants in CLI argument definitions to avoid hardcoding strings:

		WorkingGroupOption = Annotated[
		list[str] \| None,
		typer.Option("--working-group", envvar=ConfigEnvVar.TDC_WORKING_GROUP),
		typer.Option("--working-group", envvar=ConfigEnvVar.TDC_WORKING_GROUP.name),
		]
		"""

		from __future__ import annotations

		import logging
		from enum import StrEnum

		logger = logging.getLogger(__name__)


		class ConfigEnvVar(StrEnum):
		"""Environment variable to TOML config field path mappings.
		@@ -101,38 +97,10 @@ class ConfigEnvVar(StrEnum):
		TDC_AI_BATCH_SIZE = "ai.batch_size"


		# Derived dict for backward compatibility (tools/docs that need dict form)
		ENV_VAR_MAPPINGS: dict[str, str] = {e.name: e.value for e in ConfigEnvVar}


		# Deprecated environment variables that will produce warnings.
		# Format: "OLD_VAR": "Use NEW_VAR instead"
		DEPRECATED_ENV_VARS: dict[str, str] = {}


		def log_deprecation_warning(env_var_name: str) -> None:
		"""Log a warning for a deprecated environment variable.

		Args:
		env_var_name: Name of the deprecated environment variable.
		"""
		if env_var_name in DEPRECATED_ENV_VARS:
		replacement = DEPRECATED_ENV_VARS[env_var_name]
		logger.warning(
		"Environment variable '%s' is deprecated. %s",
		env_var_name,
		replacement,
		)
		else:
		logger.debug(
		"Environment variable '%s' is set but has no documented mapping",
		env_var_name,
		)
		TOML_PATH_TO_ENV_VAR: dict[str, str] = {e.value: e.name for e in ConfigEnvVar}


		__all__ = [
		"DEPRECATED_ENV_VARS",
		"ENV_VAR_MAPPINGS",
		"TOML_PATH_TO_ENV_VAR",
		"ConfigEnvVar",
		"log_deprecation_warning",
		]

src/tdoc_crawler/config/export.py

+4 −4

Original line number	Diff line number	Diff line
		"""Export TDocCrawlerConfig to various file formats with documentation."""
		"""Export ThreeGPPConfig to various file formats with documentation."""

		from __future__ import annotations

		@@ -8,7 +8,7 @@ from typing import Any, Literal

		import yaml

		from tdoc_crawler.config.settings import TDocCrawlerConfig
		from tdoc_crawler.config.settings import ThreeGPPConfig

		FormatType = Literal["toml", "yaml", "json"]

		@@ -24,9 +24,9 @@ def _default_value_for_field(info: Any) -> Any:
		class ConfigExporter:
		"""Export configuration to file formats with documentation comments."""

		def __init__(self, config: TDocCrawlerConfig \| None = None):
		def __init__(self, config: ThreeGPPConfig \| None = None):
		"""Initialize with config instance (or create with defaults)."""
		self.config = config or TDocCrawlerConfig()
		self.config = config or ThreeGPPConfig()

		def export(self, format: FormatType = "toml") -> str:
		"""Export config to string in specified format with comments."""

src/tdoc_crawler/config/service.py

deleted100644 → 0

+0 −80

Original line number	Diff line number	Diff line
		"""Unified configuration service providing access to all config types."""

		from __future__ import annotations

		from pathlib import Path

		from tdoc_crawler.config import CacheManager, resolve_cache_manager
		from tdoc_crawler.models.base import HttpCacheConfig
		from tdoc_crawler.models.crawl_limits import CrawlLimits


		class ConfigService:
		"""Unified access point for all application configuration.

		Provides lazy access to cache settings, HTTP cache settings,
		AI settings, and crawl limits. Integrates with the existing
		CacheManager registry and ServiceContainer pattern.

		Example:
		config = ConfigService()
		http_config = config.http_cache
		crawl_limits = config.crawl_limits
		"""

		def __init__(
		self,
		cache_manager_name: str \| None = None,
		cache_dir: Path \| None = None,
		) -> None:
		"""Initialize ConfigService.

		Args:
		cache_manager_name: Name of an already-registered CacheManager to use.
		If None, uses the 'default' manager when available, or creates one
		from environment variables.
		cache_dir: Explicit cache root path. Only used if no CacheManager is
		registered under cache_manager_name. Takes precedence over env vars.
		"""
		self._cache_manager_name = cache_manager_name
		self._cache_dir = cache_dir
		self._http_cache: HttpCacheConfig \| None = None
		self._crawl_limits: CrawlLimits \| None = None

		@property
		def cache_manager(self) -> CacheManager:
		"""Return the resolved CacheManager instance.

		Returns:
		The registered CacheManager for this service's name, or the
		default manager if no name was provided.

		Raises:
		ValueError: If no matching CacheManager is registered and no
		cache_dir was provided to create one.
		"""
		try:
		return resolve_cache_manager(self._cache_manager_name)
		except ValueError:
		# No registered manager - create one from cache_dir or env
		manager = CacheManager(
		root_path=self._cache_dir,
		name=self._cache_manager_name or "default",
		ensure_paths=True,
		)
		manager.register(force=True)
		return manager

		@property
		def http_cache(self) -> HttpCacheConfig:
		"""Return HTTP cache configuration resolved from env vars and defaults.

		Returns:
		HttpCacheConfig instance with resolved TTL and refresh settings.
		"""
		if self._http_cache is None:
		self._http_cache = HttpCacheConfig.resolve_http_cache_config(cache_file=self.cache_manager.http_cache_file)
		return self._http_cache


		__all__ = ["ConfigService"]

src/tdoc_crawler/config/settings.py

+59 −45

Original line number	Diff line number	Diff line
		"""Configuration settings for TDocCrawler using pydantic-settings.
		"""Configuration settings for 3GPP Crawler using pydantic-settings.

		This module provides the TDocCrawlerConfig class with nested models for:
		This module provides the ThreeGPPConfig class with nested models for:
		- PathConfig: File system paths (cache, checkout, AI store)
		- HttpConfig: HTTP caching and request behavior
		- CredentialsConfig: Portal authentication credentials
		- CrawlConfig: Crawling behavior and filters

		All paths default to values from CacheManager but can be overridden via
		All paths default to sensible defaults but can be overridden via
		environment variables with the TDC_ prefix.
		"""

		from __future__ import annotations

		from importlib import import_module
		from pathlib import Path

		from pydantic import AliasChoices, Field, field_validator
		from pydantic_settings import BaseSettings, SettingsConfigDict

		# Default values matching those in __init__.py
		from tdoc_crawler.config.env_vars import ConfigEnvVar
		from tdoc_crawler.config.sources import discover_config_files, load_config_file, merge_configs

		_DEFAULT_CACHE_DIR = Path.home() / ".3gpp-crawler"
		_DEFAULT_DATABASE_FILENAME = "3gpp_crawler.db"
		_DEFAULT_HTTP_CACHE_FILENAME = "http-cache.sqlite3"
		_DEFAULT_CHECKOUT_DIRNAME = "checkout"
		_DEFAULT_AI_CACHE_DIRNAME = "lightrag"

		WORKSPACE_REGISTRY_FILENAME = "workspaces.json"


		class PathConfig(BaseSettings):
		"""File system path configuration.
		@@ -37,7 +40,7 @@ class PathConfig(BaseSettings):

		cache_dir: Path = Field(
		default=_DEFAULT_CACHE_DIR,
		validation_alias=AliasChoices("TDC_CACHE_DIR", "cache_dir"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_CACHE_DIR.name, "cache_dir"),
		description="Root cache directory for storing downloaded files and metadata",
		)
		db_filename: str = Field(
		@@ -73,15 +76,13 @@ class PathConfig(BaseSettings):
		"""Path to the AI cache directory for embeddings and graphs."""
		return self.cache_dir / _DEFAULT_AI_CACHE_DIRNAME

		def ai_embed_dir(self, embedding_model: str) -> Path:
		"""Path to store AI-related files for a specific embedding model.

		Args:
		embedding_model: The embedding model identifier.
		@property
		def ai_workspace_file(self) -> Path:
		"""Path to the workspace registry file."""
		return self.ai_cache_dir / WORKSPACE_REGISTRY_FILENAME

		Returns:
		Path to the model-specific AI directory.
		"""
		def ai_embed_dir(self, embedding_model: str) -> Path:
		"""Path to store AI-related files for a specific embedding model."""
		return (self.ai_cache_dir / embedding_model).resolve()

		@field_validator("cache_dir", mode="before")
		@@ -107,34 +108,34 @@ class HttpConfig(BaseSettings):
		cache_ttl: int = Field(
		default=7200,
		ge=0,
		validation_alias=AliasChoices("HTTP_CACHE_TTL", "cache_ttl"),
		validation_alias=AliasChoices(ConfigEnvVar.HTTP_CACHE_TTL.name, "cache_ttl"),
		description="Time-to-live for HTTP cache entries in seconds",
		)
		cache_enabled: bool = Field(
		default=True,
		validation_alias=AliasChoices("HTTP_CACHE_ENABLED", "cache_enabled"),
		validation_alias=AliasChoices(ConfigEnvVar.HTTP_CACHE_ENABLED.name, "cache_enabled"),
		description="Enable HTTP response caching",
		)
		cache_refresh_on_access: bool = Field(
		default=True,
		validation_alias=AliasChoices("HTTP_CACHE_REFRESH_ON_ACCESS", "cache_refresh_on_access"),
		validation_alias=AliasChoices(ConfigEnvVar.HTTP_CACHE_REFRESH_ON_ACCESS.name, "cache_refresh_on_access"),
		description="Refresh cache TTL on each access",
		)
		verify_ssl: bool = Field(
		default=True,
		validation_alias=AliasChoices("TDC_VERIFY_SSL", "verify_ssl"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_VERIFY_SSL.name, "verify_ssl"),
		description="Verify SSL certificates for HTTPS requests",
		)
		max_retries: int = Field(
		default=3,
		ge=0,
		validation_alias=AliasChoices("TDC_MAX_RETRIES", "max_retries"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_MAX_RETRIES.name, "max_retries"),
		description="Maximum number of retry attempts for failed requests",
		)
		timeout: int = Field(
		default=30,
		ge=1,
		validation_alias=AliasChoices("TDC_TIMEOUT", "timeout"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_TIMEOUT.name, "timeout"),
		description="HTTP request timeout in seconds",
		)

		@@ -143,7 +144,7 @@ class HttpConfig(BaseSettings):
		def _parse_int(cls, value: str \| int \| None) -> int:
		"""Parse integer values from environment strings."""
		if value is None:
		return 0 if cls.__name__ in ("HttpConfig",) else 0
		return 0
		if isinstance(value, str):
		value = value.strip()
		return int(value) if value else 0
		@@ -173,17 +174,17 @@ class CredentialsConfig(BaseSettings):

		username: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_EOL_USERNAME", "username"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_EOL_USERNAME.name, "username"),
		description="Username for ETSI Online (EOL) portal authentication",
		)
		password: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_EOL_PASSWORD", "password"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_EOL_PASSWORD.name, "password"),
		description="Password for ETSI Online (EOL) portal authentication",
		)
		prompt: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_EOL_PROMPT", "prompt"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_EOL_PROMPT.name, "prompt"),
		description="Custom prompt message for interactive credential entry",
		)

		@@ -204,56 +205,70 @@ class CrawlConfig(BaseSettings):

		working_group: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_WORKING_GROUP", "working_group"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_WORKING_GROUP.name, "working_group"),
		description="Filter by working group (e.g., S4, RAN1, CT3)",
		)
		sub_group: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_SUB_GROUP", "sub_group"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_SUB_GROUP.name, "sub_group"),
		description="Filter by sub-working group",
		)
		date_start: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_START_DATE", "date_start"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_START_DATE.name, "date_start"),
		description="Start date filter (YYYY-MM-DD, YYYY-MM, or YYYY format)",
		)
		date_end: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_END_DATE", "date_end"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_END_DATE.name, "date_end"),
		description="End date filter (YYYY-MM-DD, YYYY-MM, or YYYY format)",
		)
		source_like: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_SOURCE_LIKE", "source_like"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_SOURCE_LIKE.name, "source_like"),
		description="SQL LIKE pattern to match document source",
		)
		agenda_like: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_AGENDA_LIKE", "agenda_like"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_AGENDA_LIKE.name, "agenda_like"),
		description="SQL LIKE pattern to match agenda item",
		)
		title_like: str \| None = Field(
		default=None,
		validation_alias=AliasChoices("TDC_TITLE_LIKE", "title_like"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_TITLE_LIKE.name, "title_like"),
		description="SQL LIKE pattern to match document title",
		)
		limit: int = Field(
		default=1000,
		ge=1,
		validation_alias=AliasChoices("TDC_LIMIT_TDOCS", "limit"),
		description="Maximum number of documents to crawl",
		validation_alias=AliasChoices(ConfigEnvVar.TDC_LIMIT_TDOCS.name, "limit"),
		description="Maximum number of TDocs to crawl per run",
		)
		limit_meetings: int \| None = Field(
		default=None,
		validation_alias=AliasChoices(ConfigEnvVar.TDC_LIMIT_MEETINGS.name, "limit_meetings"),
		description="Maximum meetings to crawl overall (negative = newest N)",
		)
		limit_meetings_per_subwg: int \| None = Field(
		default=None,
		validation_alias=AliasChoices(ConfigEnvVar.TDC_LIMIT_MEETINGS_PER_SUBWG.name, "limit_meetings_per_subwg"),
		description="Per sub-WG meeting limit (negative = newest N)",
		)
		limit_subwgs: int \| None = Field(
		default=None,
		description="Maximum sub-working groups to process",
		)
		workers: int = Field(
		default=4,
		ge=1,
		le=32,
		validation_alias=AliasChoices("TDC_WORKERS", "workers"),
		validation_alias=AliasChoices(ConfigEnvVar.TDC_WORKERS.name, "workers"),
		description="Number of concurrent workers for crawling",
		)


		class TDocCrawlerConfig(BaseSettings):
		"""Main configuration class for TDocCrawler.
		class ThreeGPPConfig(BaseSettings):
		"""Main configuration class for 3GPP Crawler.

		Aggregates all configuration sub-models:
		- path: File system paths
		@@ -290,8 +305,8 @@ class TDocCrawlerConfig(BaseSettings):
		cls,
		config_file: Path \| str \| None = None,
		cwd: Path \| str \| None = None,
		) -> TDocCrawlerConfig:
		"""Create a TDocCrawlerConfig instance with config file and environment values.
		) -> ThreeGPPConfig:
		"""Create a ThreeGPPConfig instance with config file and environment values.

		Config precedence (highest to lowest):
		1. CLI explicit config_file parameter
		@@ -306,14 +321,8 @@ class TDocCrawlerConfig(BaseSettings):
		Defaults to current working directory.

		Returns:
		A validated TDocCrawlerConfig instance.
		A validated ThreeGPPConfig instance.
		"""
		# Import via import_module to avoid PLC0415 and circular import
		sources = import_module("tdoc_crawler.config.sources")
		discover_config_files = sources.discover_config_files
		load_config_file = sources.load_config_file
		merge_configs = sources.merge_configs

		cwd_path = Path(cwd) if cwd is not None else Path.cwd()

		if config_file:
		@@ -327,10 +336,15 @@ class TDocCrawlerConfig(BaseSettings):
		return cls(**merged)


		# Backward-compatible alias — remove after all consumers are updated
		TDocCrawlerConfig = ThreeGPPConfig

		__all__ = [
		"WORKSPACE_REGISTRY_FILENAME",
		"CrawlConfig",
		"CredentialsConfig",
		"HttpConfig",
		"PathConfig",
		"TDocCrawlerConfig",
		"ThreeGPPConfig",
		]