Commit 6d15352f authored by Jan Reimes's avatar Jan Reimes
Browse files

Delete Pydantic duplicates of Oxyde DB models 

Remove Pydantic MeetingMetadata, Specification, SpecificationSourceRecord,
SpecificationVersion, and SpecificationDownload — all had identical or
near-identical Oxyde equivalents in database/oxyde_models.py.

Changes:
- Delete Pydantic MeetingMetadata from meetings/models.py
- Delete 4 Pydantic spec models from specs/models.py (keep dataclasses)
- Fix WHATSPEC_TDOC_URL_TEMPLATE.format() missing 'base' parameter
- Fix WHATSPEC_BASE_URL import path in test_whatthespec.py
- Add record_id to SpecificationSourceRecord/Version constructions in
  tests (required by Oxyde db_pk=True)
- Update 12 import sites to use database.oxyde_models directly
parent 069bbfab

src/tdoc_crawler/cli/crawl.py

+2 −1
Original line number Diff line number Diff line
@@ -48,11 +48,12 @@ from tdoc_crawler.cli.printing import print_spec_crawl_table, spec_crawl_to_dict 
from tdoc_crawler.config import TDocCrawlerConfig

from tdoc_crawler.credentials import set_credentials

from tdoc_crawler.database import MeetingDatabase, TDocDatabase

from tdoc_crawler.database.oxyde_models import MeetingMetadata

from tdoc_crawler.database.specs import SpecCrawlResult, SpecDatabase

from tdoc_crawler.http_client import create_cached_session

from tdoc_crawler.logging import DEFAULT_LEVEL as DEFAULT_VERBOSITY

from tdoc_crawler.logging import set_verbosity

from tdoc_crawler.meetings.models import MeetingCrawlConfig, MeetingMetadata, MeetingQueryConfig

from tdoc_crawler.meetings.models import MeetingCrawlConfig, MeetingQueryConfig

from tdoc_crawler.meetings.operations.crawl import MeetingCrawler, MeetingCrawlResult

from tdoc_crawler.models.base import OutputFormat, SortOrder

from tdoc_crawler.models.subworking_groups import SUBTB_INDEX

src/tdoc_crawler/cli/crawl/_helpers.py

+1 −1
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ from __future__ import annotations 
from datetime import date



from tdoc_crawler.cli._shared import console

from tdoc_crawler.meetings.models import MeetingMetadata

from tdoc_crawler.database.oxyde_models import MeetingMetadata

from tdoc_crawler.meetings.operations.crawl import MeetingCrawlResult

from tdoc_crawler.models.subworking_groups import SUBTB_INDEX

from tdoc_crawler.models.working_groups import WorkingGroup

src/tdoc_crawler/cli/printing.py

+1 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ from tdoc_crawler.cli.formatting import TableColumnSpec, print_structured_output 
from tdoc_crawler.database.oxyde_models import TDocMetadata

from tdoc_crawler.database.specs import SpecCrawlResult

from tdoc_crawler.logging import get_console

from tdoc_crawler.meetings.models import MeetingMetadata

from tdoc_crawler.database.oxyde_models import MeetingMetadata

from tdoc_crawler.models.base import OutputFormat

from tdoc_crawler.models.subworking_groups import SUBTB_INDEX

from tdoc_crawler.models.working_groups import WorkingGroup

src/tdoc_crawler/meetings/models.py

+2 −47
Original line number Diff line number Diff line
@@ -3,61 +3,17 @@ 
from __future__ import annotations



from collections.abc import Iterable

from datetime import date, datetime

from datetime import date



from pydantic import BaseModel, Field, field_validator, model_validator

from pydantic import BaseModel, Field, field_validator



from tdoc_crawler.config.settings import HttpConfig

from tdoc_crawler.meetings.utils import normalize_subgroup_alias

from tdoc_crawler.models.base import SortOrder

from tdoc_crawler.models.subworking_groups import SUBTB_INDEX

from tdoc_crawler.models.working_groups import WorkingGroup

from tdoc_crawler.utils.misc import utc_now

from tdoc_crawler.utils.normalization import normalize_working_group_list





class MeetingMetadata(BaseModel):

    """Persistent representation for a 3GPP meeting."""



    # parsed from the portal

    meeting_id: int = Field(..., description="Unique meeting identifier from the 3GPP portal")

    tbid: int = Field(

        ...,

        description="Technical body identifier of the parent working group (FK to working_groups.tbid)",

    )

    subtb: int | None = Field(

        ...,

        description="Sub-technical body identifier (FK to subworking_groups.subtb)",

    )

    short_name: str = Field(..., description="Short meeting name (e.g., SA4#134)")

    title: str | None = Field(None, description="Descriptive meeting title")

    start_date: date | None = Field(None, description="Meeting start date")

    end_date: date | None = Field(None, description="Meeting end date")

    location: str | None = Field(None, description="Meeting location")

    files_url: str | None = Field(None, description="HTTP directory containing meeting documents")

    portal_url: str | None = Field(None, description="Link to the portal meeting overview")



    # Crawling statistics

    tdoc_count: int = Field(0, ge=0, description="Number of TDocs associated with this meeting")



    # Database management fields

    last_synced: datetime = Field(default_factory=utc_now, description="Timestamp when the meeting was last refreshed")

    created_at: datetime = Field(default_factory=utc_now, description="Timestamp when the record was first stored")

    updated_at: datetime = Field(default_factory=utc_now, description="Timestamp when the record was last updated")



    @model_validator(mode="after")

    def _sync_relationships(self) -> MeetingMetadata:

        if self.subtb is None:

            return self

        record = SUBTB_INDEX.get(self.subtb)

        if record is None:

            return self

        if record.tbid != self.tbid:

            msg = "tbid does not match subworking group"

            raise ValueError(msg)

        return self





class MeetingCrawlConfig(BaseModel):

    """Configuration for meeting crawling operations."""
@@ -132,6 +88,5 @@ class MeetingQueryConfig(BaseModel): 


__all__ = [

    "MeetingCrawlConfig",

    "MeetingMetadata",

    "MeetingQueryConfig",

]

src/tdoc_crawler/meetings/operations/crawl.py

+2 −1
Original line number Diff line number Diff line
@@ -7,8 +7,9 @@ from collections.abc import Callable 
from dataclasses import dataclass



from tdoc_crawler.database.meetings import MeetingDatabase

from tdoc_crawler.database.oxyde_models import MeetingMetadata

from tdoc_crawler.logging import get_logger

from tdoc_crawler.meetings.models import MeetingCrawlConfig, MeetingMetadata

from tdoc_crawler.meetings.models import MeetingCrawlConfig

from tdoc_crawler.meetings.sources.portal import fetch_meetings

from tdoc_crawler.models.working_groups import WorkingGroup