Commit d80e4a95 authored by Jan Reimes's avatar Jan Reimes
Browse files

fix(cli): handle None URL in prepare_tdoc_file and improve error handling 

* Raise ValueError if metadata.url is None to prevent further issues.
* Ensure filename is converted to string for consistency.
* Add comments for clarity on non-zip file handling.
parent 4594247d

.vscode/settings.json

+1 −2
Original line number Diff line number Diff line
@@ -15,6 +15,5 @@ 
    "chat.tools.terminal.autoApprove": {

        ".specify/scripts/bash/": true,

        ".specify/scripts/powershell/": true

    },

    "beads.pathToBd": "${env:VIRTUAL_ENV}/scripts/bd.exe"

    }

}

src/tdoc_crawler/cli/helpers.py

+6 −3
Original line number Diff line number Diff line
@@ -314,13 +314,15 @@ def prepare_tdoc_file(metadata: TDocMetadata, cache_dir: Path, return_dir: bool 
    Returns:

        Path to the downloaded file, or the extract directory when return_dir is True.

    """

    # Handle the case where metadata.url is None

    if metadata.url is None:

        raise ValueError(f"Cannot prepare TDoc file for {metadata.tdoc_id}: URL is None")



    downloads_dir = cache_dir / "checkout"

    downloads_dir.mkdir(parents=True, exist_ok=True)

    path = urlparse(metadata.url).path

    filename = posixpath.basename(path)

    filename = str(posixpath.basename(path))



    # TODO: unnecessary check for suffix here, download_to_path() is called in any case (should be called just once)?

    # TODO: Only extraction of tdoc is optional/if it is a zip, but not the download itself.

    suffix = Path(filename).suffix.lower()



    if suffix == ".zip":
@@ -343,6 +345,7 @@ def prepare_tdoc_file(metadata: TDocMetadata, cache_dir: Path, return_dir: bool 
            raise FileNotFoundError("no-files-in-archive")

        return extract_dir if return_dir else files[0]



    # For non-zip files, download directly

    target_suffix = suffix or ""

    target_name = filename if filename else f"{metadata.tdoc_id}{target_suffix or '.bin'}"

    target_path = downloads_dir / target_name

src/tdoc_crawler/models/__init__.py

+31 −10
Original line number Diff line number Diff line
@@ -3,18 +3,39 @@ 
from __future__ import annotations



# Re-export all public symbols

from .base import BaseConfigModel  # noqa: F401

from .base import DEFAULT_CACHE_DIR, HttpCacheConfig, OutputFormat, PortalCredentials, SortOrder, utc_now

from .base import (

    DEFAULT_CACHE_DIR,

    BaseConfigModel,  # noqa: F401

    HttpCacheConfig,

    OutputFormat,

    PortalCredentials,

    SortOrder,

    utc_now,

)

from .crawl_limits import CrawlLimits  # noqa: F401

from .crawl_log import CrawlLogEntry  # noqa: F401

from .meetings import MeetingMetadata  # noqa: F401

from .meetings import MeetingCrawlConfig, MeetingQueryConfig

from .subworking_groups import SUBTB_INDEX  # noqa: F401

from .subworking_groups import CODE_INDEX, SUBWORKING_GROUP_RECORDS, SubWorkingGroupRecord

from .tdocs import TDocCrawlConfig  # noqa: F401

from .tdocs import CrawlConfig, QueryConfig, TDocMetadata

from .working_groups import WorkingGroup  # noqa: F401

from .working_groups import WORKING_GROUP_RECORDS, WorkingGroupRecord

from .meetings import (

    MeetingCrawlConfig,

    MeetingMetadata,  # noqa: F401

    MeetingQueryConfig,

)

from .subworking_groups import (

    CODE_INDEX,

    SUBTB_INDEX,  # noqa: F401

    SUBWORKING_GROUP_RECORDS,

    SubWorkingGroupRecord,

)

from .tdocs import (

    CrawlConfig,

    QueryConfig,

    TDocCrawlConfig,  # noqa: F401

    TDocMetadata,

)

from .working_groups import (

    WORKING_GROUP_RECORDS,

    WorkingGroup,  # noqa: F401

    WorkingGroupRecord,

)



__all__ = [

    "CODE_INDEX",

src/tdoc_crawler/specs/downloads.py

+14 −10
Original line number Diff line number Diff line
@@ -100,18 +100,22 @@ class SpecDownloads: 


        versions.sort(key=lambda x: parse_version(x.version), reverse=True)



        # If specific release requested, filter?

        # Usually 'release' maps to strict major version or Rel-XX.

        # "17" -> 17.x.x.

        # For now, I'll ignore complex release filtering unless 'latest' is not used.

        # If release != "latest", ideally we match Rel-{release}.

        # Existing logic in plan said: "when a non-default value is provided, it must match metadata versions".

        # If specific release requested, filter versions by major version

        if release != "latest":

            try:

                release_major = int(release.split(".")[0])

                # Filter versions that match this major release

                filtered_versions = [v for v in versions if parse_version(v.version)[0] == release_major]

                if filtered_versions:

                    versions = filtered_versions

                else:

                    msg = f"No versions found for spec {normalized} with release {release}"

                    raise ValueError(msg)

            except (ValueError, IndexError) as e:

                msg = f"Invalid release format: {release}. Expected format like '17' or '17.1'"

                raise ValueError(msg) from e



        target = versions[0]

        if release != "latest":

            # Simple match check

            # Assuming release matches version prefix or some field

            pass  # TODO: Implement strict release filtering



        # Construct 3GPP FTP URL

        series = f"{normalized.split('.', maxsplit=1)[0]}_series"