Loading src/tdoc_crawler/models/base.py +35 −22 Original line number Diff line number Diff line Loading @@ -6,7 +6,7 @@ import os from dataclasses import dataclass from enum import StrEnum from pydantic import BaseModel from pydantic import BaseModel, Field class OutputFormat(StrEnum): Loading @@ -23,35 +23,28 @@ class SortOrder(StrEnum): ASC = "asc" DESC = "desc" class BaseConfigModel(BaseModel): """Shared configuration base enabling attribute parsing and whitespace handling.""" model_config = {"str_strip_whitespace": True, "use_enum_values": False} @dataclass class PortalCredentials: """Credentials required for ETSI Online Account (EOL) protected resources.""" username: str password: str _DEFAULT_TTL: int = 7200 _DEFAULT_TTL_ON_ACCESS: bool = True _DEFAULT_MAX_RETRIES: int = 3 @dataclass class HttpCacheConfig: """HTTP cache configuration for hishel caching.""" ttl: int = 7200 refresh_ttl_on_access: bool = True ttl: int refresh_ttl_on_access: bool max_retries: int @classmethod def resolve_http_cache_config(cls, cache_ttl: int | None = None, cache_refresh_on_access: bool | None = None) -> HttpCacheConfig: def resolve_http_cache_config( cls, cache_ttl: int | None = None, cache_refresh_on_access: bool | None = None, max_retries: int | None = None ) -> HttpCacheConfig: """Resolve HTTP cache configuration from CLI parameters and environment variables. Args: cache_ttl: TTL for cache entries (CLI parameter) cache_refresh_on_access: Whether to refresh TTL on access (CLI parameter) max_retries: Maximum number of retry attempts for failed requests (CLI parameter) Returns: HttpCacheConfig instance with resolved values Loading @@ -61,16 +54,36 @@ class HttpCacheConfig: ttl = cache_ttl else: env_ttl = os.getenv("HTTP_CACHE_TTL") ttl = int(env_ttl) if env_ttl else 7200 ttl = int(env_ttl) if env_ttl else _DEFAULT_TTL # Handle refresh on access - check CLI param, then env var, then default if cache_refresh_on_access is not None: refresh_on_access = cache_refresh_on_access elif env_refresh := os.getenv("HTTP_CACHE_REFRESH_ON_ACCESS"): env_refresh = env_refresh.lower() refresh_on_access = env_refresh in ("true", "1", "yes", "on", "t", "y") else: env_refresh = os.getenv("HTTP_CACHE_REFRESH_ON_ACCESS", "").lower() refresh_on_access = env_refresh in ("true", "1", "yes", "on", "t", "y") if env_refresh else True refresh_on_access = _DEFAULT_TTL_ON_ACCESS max_retries = max_retries or _DEFAULT_MAX_RETRIES return HttpCacheConfig(ttl=ttl, refresh_ttl_on_access=refresh_on_access, max_retries=max_retries) return HttpCacheConfig(ttl=ttl, refresh_ttl_on_access=refresh_on_access) class BaseConfigModel(BaseModel): """Shared configuration base enabling attribute parsing and whitespace handling.""" model_config = {"str_strip_whitespace": True, "use_enum_values": False} cache_manager_name: str | None = Field(default=None, description="Cache configuration manager name") http_cache: HttpCacheConfig = Field(default_factory=HttpCacheConfig.resolve_http_cache_config, description="HTTP cache configuration") @dataclass class PortalCredentials: """Credentials required for ETSI Online Account (EOL) protected resources.""" username: str password: str __all__ = [ Loading Loading
src/tdoc_crawler/models/base.py +35 −22 Original line number Diff line number Diff line Loading @@ -6,7 +6,7 @@ import os from dataclasses import dataclass from enum import StrEnum from pydantic import BaseModel from pydantic import BaseModel, Field class OutputFormat(StrEnum): Loading @@ -23,35 +23,28 @@ class SortOrder(StrEnum): ASC = "asc" DESC = "desc" class BaseConfigModel(BaseModel): """Shared configuration base enabling attribute parsing and whitespace handling.""" model_config = {"str_strip_whitespace": True, "use_enum_values": False} @dataclass class PortalCredentials: """Credentials required for ETSI Online Account (EOL) protected resources.""" username: str password: str _DEFAULT_TTL: int = 7200 _DEFAULT_TTL_ON_ACCESS: bool = True _DEFAULT_MAX_RETRIES: int = 3 @dataclass class HttpCacheConfig: """HTTP cache configuration for hishel caching.""" ttl: int = 7200 refresh_ttl_on_access: bool = True ttl: int refresh_ttl_on_access: bool max_retries: int @classmethod def resolve_http_cache_config(cls, cache_ttl: int | None = None, cache_refresh_on_access: bool | None = None) -> HttpCacheConfig: def resolve_http_cache_config( cls, cache_ttl: int | None = None, cache_refresh_on_access: bool | None = None, max_retries: int | None = None ) -> HttpCacheConfig: """Resolve HTTP cache configuration from CLI parameters and environment variables. Args: cache_ttl: TTL for cache entries (CLI parameter) cache_refresh_on_access: Whether to refresh TTL on access (CLI parameter) max_retries: Maximum number of retry attempts for failed requests (CLI parameter) Returns: HttpCacheConfig instance with resolved values Loading @@ -61,16 +54,36 @@ class HttpCacheConfig: ttl = cache_ttl else: env_ttl = os.getenv("HTTP_CACHE_TTL") ttl = int(env_ttl) if env_ttl else 7200 ttl = int(env_ttl) if env_ttl else _DEFAULT_TTL # Handle refresh on access - check CLI param, then env var, then default if cache_refresh_on_access is not None: refresh_on_access = cache_refresh_on_access elif env_refresh := os.getenv("HTTP_CACHE_REFRESH_ON_ACCESS"): env_refresh = env_refresh.lower() refresh_on_access = env_refresh in ("true", "1", "yes", "on", "t", "y") else: env_refresh = os.getenv("HTTP_CACHE_REFRESH_ON_ACCESS", "").lower() refresh_on_access = env_refresh in ("true", "1", "yes", "on", "t", "y") if env_refresh else True refresh_on_access = _DEFAULT_TTL_ON_ACCESS max_retries = max_retries or _DEFAULT_MAX_RETRIES return HttpCacheConfig(ttl=ttl, refresh_ttl_on_access=refresh_on_access, max_retries=max_retries) return HttpCacheConfig(ttl=ttl, refresh_ttl_on_access=refresh_on_access) class BaseConfigModel(BaseModel): """Shared configuration base enabling attribute parsing and whitespace handling.""" model_config = {"str_strip_whitespace": True, "use_enum_values": False} cache_manager_name: str | None = Field(default=None, description="Cache configuration manager name") http_cache: HttpCacheConfig = Field(default_factory=HttpCacheConfig.resolve_http_cache_config, description="HTTP cache configuration") @dataclass class PortalCredentials: """Credentials required for ETSI Online Account (EOL) protected resources.""" username: str password: str __all__ = [ Loading