Commit d5f1d710 authored by Jan Reimes's avatar Jan Reimes
Browse files

♻️ refactor(database): replace logging with custom logger across modules 

- Removed standard logging imports and replaced with custom logger.
- Updated logger initialization in database-related files.
- Improved consistency in logging approach throughout the database module.
parent 4c551a7f

src/tdoc_crawler/database/__init__.py

+0 −5
Original line number Diff line number Diff line
@@ -2,17 +2,12 @@ 


from __future__ import annotations



import logging



from tdoc_crawler.database.base import DocDatabase

from tdoc_crawler.database.errors import DatabaseError

from tdoc_crawler.database.meetings import MeetingDatabase

from tdoc_crawler.database.specs import SpecDatabase

from tdoc_crawler.database.tdocs import TDocDatabase



logger = logging.getLogger(__name__)





__all__ = [

    "DatabaseError",

    "DocDatabase",

src/tdoc_crawler/database/base.py

+5 −4
Original line number Diff line number Diff line 
"""Base database class with common operations."""



import json

import logging

from collections.abc import Iterable

from datetime import datetime

from pathlib import Path
@@ -10,6 +9,7 @@ from typing import Self 
from pydantic_sqlite import DataBase



from tdoc_crawler.database.errors import DatabaseError

from tdoc_crawler.logging import get_logger

from tdoc_crawler.meetings.models import MeetingMetadata

from tdoc_crawler.models.crawl_log import CrawlLogEntry

from tdoc_crawler.models.subworking_groups import SUBWORKING_GROUP_RECORDS
@@ -22,7 +22,7 @@ from tdoc_crawler.specs.models import ( 
)

from tdoc_crawler.tdocs.models import TDocMetadata



_logger = logging.getLogger(__name__)

_logger = get_logger(__name__)





class DocDatabase:
@@ -143,13 +143,14 @@ class DocDatabase: 
            items_updated: Number of existing items updated

            errors_count: Number of errors encountered

        """

        # DataBase._db is a sqlite3.Connection

        # Use the pydantic-sqlite connection to execute raw SQL

        db = self.connection._db

        db.execute(

            "UPDATE crawl_log SET end_time = ?, items_added = ?, items_updated = ?, errors_count = ?, status = ? WHERE log_id = ?",

            (datetime.now().isoformat(), items_added, items_updated, errors_count, "COMPLETED", crawl_id),

        )

        db.commit()

        # pydantic-sqlite handles commits automatically via context manager

        # No explicit commit needed



    def _ensure_reference_data(self) -> None:

        """Populate reference tables for working and subworking groups."""

src/tdoc_crawler/database/specs.py

+3 −2
Original line number Diff line number Diff line
@@ -4,12 +4,12 @@ from __future__ import annotations 


import contextlib

import json

import logging

from collections import defaultdict

from dataclasses import dataclass

from datetime import datetime



from tdoc_crawler.database.base import DocDatabase

from tdoc_crawler.logging import get_logger

from tdoc_crawler.specs.models import (

    Specification,

    SpecificationDownload,
@@ -18,9 +18,10 @@ from tdoc_crawler.specs.models import ( 
    SpecQueryFilters,

    SpecQueryResult,

)

from tdoc_crawler.specs.sources.base import SpecSource

from tdoc_crawler.utils.normalization import normalize_spec_number



_logger = logging.getLogger(__name__)

_logger = get_logger(__name__)





@dataclass(frozen=True)

src/tdoc_crawler/database/tdocs.py

+5 −7
Original line number Diff line number Diff line 
"""TDoc database operations."""



import logging

from collections.abc import Callable, Iterable

from datetime import UTC, datetime

from decimal import Decimal



from tdoc_crawler.database.base import DocDatabase

from tdoc_crawler.database.meetings import MeetingDatabase

from tdoc_crawler.logging import get_logger

from tdoc_crawler.models import WorkingGroup

from tdoc_crawler.tdocs.models import QueryConfig, TDocMetadata

from tdoc_crawler.tdocs.models import TDocMetadata, TDocQueryConfig

from tdoc_crawler.utils.misc import utc_now



_logger = logging.getLogger(__name__)

_logger = get_logger(__name__)





class TDocDatabase(DocDatabase):
@@ -85,7 +86,7 @@ class TDocDatabase(DocDatabase): 
                progress_callback(float(index), total)

        return inserted, updated



    def query_tdocs(self, config: QueryConfig) -> list[TDocMetadata]:

    def query_tdocs(self, config: TDocQueryConfig) -> list[TDocMetadata]:

        """Query TDocs with filtering and sorting.



        Args:
@@ -98,9 +99,6 @@ class TDocDatabase(DocDatabase): 
        if not records:

            return []



        # Import here to avoid circular dependency

        from tdoc_crawler.database.meetings import MeetingDatabase



        meeting_db = MeetingDatabase(self.db_file)

        meeting_db._database = self._database  # Share connection

        meeting_map = meeting_db._meeting_map()

src/tdoc_crawler/models/base.py

+2 −0
Original line number Diff line number Diff line
@@ -23,10 +23,12 @@ class SortOrder(StrEnum): 
    ASC = "asc"

    DESC = "desc"





_DEFAULT_TTL: int = 7200

_DEFAULT_TTL_ON_ACCESS: bool = True

_DEFAULT_MAX_RETRIES: int = 3





@dataclass

class HttpCacheConfig:

    """HTTP cache configuration for hishel caching."""