Commit d5f1d710 authored by Jan Reimes's avatar Jan Reimes
Browse files

♻️ refactor(database): replace logging with custom logger across modules

- Removed standard logging imports and replaced with custom logger.
- Updated logger initialization in database-related files.
- Improved consistency in logging approach throughout the database module.
parent 4c551a7f
Loading
Loading
Loading
Loading
+0 −5
Original line number Diff line number Diff line
@@ -2,17 +2,12 @@

from __future__ import annotations

import logging

from tdoc_crawler.database.base import DocDatabase
from tdoc_crawler.database.errors import DatabaseError
from tdoc_crawler.database.meetings import MeetingDatabase
from tdoc_crawler.database.specs import SpecDatabase
from tdoc_crawler.database.tdocs import TDocDatabase

logger = logging.getLogger(__name__)


__all__ = [
    "DatabaseError",
    "DocDatabase",
+5 −4
Original line number Diff line number Diff line
"""Base database class with common operations."""

import json
import logging
from collections.abc import Iterable
from datetime import datetime
from pathlib import Path
@@ -10,6 +9,7 @@ from typing import Self
from pydantic_sqlite import DataBase

from tdoc_crawler.database.errors import DatabaseError
from tdoc_crawler.logging import get_logger
from tdoc_crawler.meetings.models import MeetingMetadata
from tdoc_crawler.models.crawl_log import CrawlLogEntry
from tdoc_crawler.models.subworking_groups import SUBWORKING_GROUP_RECORDS
@@ -22,7 +22,7 @@ from tdoc_crawler.specs.models import (
)
from tdoc_crawler.tdocs.models import TDocMetadata

_logger = logging.getLogger(__name__)
_logger = get_logger(__name__)


class DocDatabase:
@@ -143,13 +143,14 @@ class DocDatabase:
            items_updated: Number of existing items updated
            errors_count: Number of errors encountered
        """
        # DataBase._db is a sqlite3.Connection
        # Use the pydantic-sqlite connection to execute raw SQL
        db = self.connection._db
        db.execute(
            "UPDATE crawl_log SET end_time = ?, items_added = ?, items_updated = ?, errors_count = ?, status = ? WHERE log_id = ?",
            (datetime.now().isoformat(), items_added, items_updated, errors_count, "COMPLETED", crawl_id),
        )
        db.commit()
        # pydantic-sqlite handles commits automatically via context manager
        # No explicit commit needed

    def _ensure_reference_data(self) -> None:
        """Populate reference tables for working and subworking groups."""
+3 −2
Original line number Diff line number Diff line
@@ -4,12 +4,12 @@ from __future__ import annotations

import contextlib
import json
import logging
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime

from tdoc_crawler.database.base import DocDatabase
from tdoc_crawler.logging import get_logger
from tdoc_crawler.specs.models import (
    Specification,
    SpecificationDownload,
@@ -18,9 +18,10 @@ from tdoc_crawler.specs.models import (
    SpecQueryFilters,
    SpecQueryResult,
)
from tdoc_crawler.specs.sources.base import SpecSource
from tdoc_crawler.utils.normalization import normalize_spec_number

_logger = logging.getLogger(__name__)
_logger = get_logger(__name__)


@dataclass(frozen=True)
+5 −7
Original line number Diff line number Diff line
"""TDoc database operations."""

import logging
from collections.abc import Callable, Iterable
from datetime import UTC, datetime
from decimal import Decimal

from tdoc_crawler.database.base import DocDatabase
from tdoc_crawler.database.meetings import MeetingDatabase
from tdoc_crawler.logging import get_logger
from tdoc_crawler.models import WorkingGroup
from tdoc_crawler.tdocs.models import QueryConfig, TDocMetadata
from tdoc_crawler.tdocs.models import TDocMetadata, TDocQueryConfig
from tdoc_crawler.utils.misc import utc_now

_logger = logging.getLogger(__name__)
_logger = get_logger(__name__)


class TDocDatabase(DocDatabase):
@@ -85,7 +86,7 @@ class TDocDatabase(DocDatabase):
                progress_callback(float(index), total)
        return inserted, updated

    def query_tdocs(self, config: QueryConfig) -> list[TDocMetadata]:
    def query_tdocs(self, config: TDocQueryConfig) -> list[TDocMetadata]:
        """Query TDocs with filtering and sorting.

        Args:
@@ -98,9 +99,6 @@ class TDocDatabase(DocDatabase):
        if not records:
            return []

        # Import here to avoid circular dependency
        from tdoc_crawler.database.meetings import MeetingDatabase

        meeting_db = MeetingDatabase(self.db_file)
        meeting_db._database = self._database  # Share connection
        meeting_map = meeting_db._meeting_map()
+2 −0
Original line number Diff line number Diff line
@@ -23,10 +23,12 @@ class SortOrder(StrEnum):
    ASC = "asc"
    DESC = "desc"


_DEFAULT_TTL: int = 7200
_DEFAULT_TTL_ON_ACCESS: bool = True
_DEFAULT_MAX_RETRIES: int = 3


@dataclass
class HttpCacheConfig:
    """HTTP cache configuration for hishel caching."""
Loading