Commit 1e43ed37 authored by Jan Reimes's avatar Jan Reimes
Browse files

Fix nested SpecDatabase pool conflict and Unicode console errors 

- DocDatabase.__aexit__: suppress RuntimeError from close_pool when
  nested overwrite already destroyed the Rust pool
- _auto_crawl_and_resolve: accept optional db param to reuse caller's
  open connection instead of opening a second SpecDatabase (root cause
  of 'Pool default not found')
- Replace Unicode arrows/checkmarks (→ ✓ ✗) with ASCII equivalents
  (-> ok FAIL) to avoid UnicodeEncodeError on Windows cp1252 console
parent e4c1f88d

demo.bat

+3 −3
Original line number Diff line number Diff line
@@ -3,13 +3,13 @@ cls 
call .venv\scripts\activate.bat



:: Crawl *all* meetings from SA4

tdoc-crawler crawl-meetings -s S4

:: tdoc-crawler crawl-meetings -s S4



:: crawl all tdocs metadata from 2016 on onwards

tdoc-crawler crawl --start-date 2016

:: tdoc-crawler crawl --start-date 2016



:: query tdocs from atias agenda items from 2018 on onwards

tdoc-crawler query --agenda "*atias*" --start-date 2018

:: tdoc-crawler query --agenda "*atias*" --start-date 2018



:: remove and re-create workspace 'atias'

3gpp-crawler workspace deactivate

src/tdoc_crawler/cli/tdoc_app.py

+2 −2
Original line number Diff line number Diff line
@@ -210,12 +210,12 @@ def checkout( 
                try:

                    checkout_path = checkout_tdoc(metadata, checkout_dir, force=force, session=session)



                    progress.console.print(f"[green]✓ {metadata.tdoc_id} → {checkout_path}")

                    progress.console.print(f"[green]ok {metadata.tdoc_id} -> {checkout_path}")



                    success_count += 1



                except (FileNotFoundError, OSError, ValueError, zipfile.BadZipFile) as exc:

                    progress.console.print(f"[red]✗ {metadata.tdoc_id}: {exc}")

                    progress.console.print(f"[red]FAIL {metadata.tdoc_id}: {exc}")



                    error_count += 1

src/tdoc_crawler/cli/workspace/members.py

+1 −1
Original line number Diff line number Diff line
@@ -81,7 +81,7 @@ def _resolve_spec_release_for_add( 
        resolved, _ = asyncio.run(

            resolve_spec_release_from_db(item, release, auto_crawl=auto_crawl),

        )

        console.print(f"[dim]  Resolved {item} release '{release}' → {resolved}[/dim]")

        console.print(f"[dim]  Resolved {item} release '{release}' -> {resolved}[/dim]")

        return resolved

    except ValueError as exc:

        console.print(f"[yellow]  Warning: {exc}. Using unresolved release '{release}'.[/yellow]")

src/tdoc_crawler/database/base.py

+4 −1
Original line number Diff line number Diff line
@@ -110,7 +110,10 @@ class DocDatabase: 


    async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: object | None) -> None:

        if self._database is not None:

            try:

                await self._database.disconnect()

            except RuntimeError:

                pass  # Pool already closed by nested overwrite

        self._database = None



    # ------------------------------------------------------------------

src/tdoc_crawler/workspaces/utils.py

+14 −9
Original line number Diff line number Diff line
@@ -120,8 +120,8 @@ async def resolve_spec_release_from_db( 
            resolved, version_codes = await _auto_crawl_and_resolve(

                normalized_spec,

                requested_release,

                db_path,

                db_file,

                db_path=db_path,

                db_file=db_file,

            )

        return resolved, version_codes
@@ -132,8 +132,8 @@ async def resolve_spec_release_from_db( 
            return await _auto_crawl_and_resolve(

                normalized_spec,

                requested_release,

                db_path,

                db_file,

                db_path=db_path,

                db_file=db_file,

            )

        return resolved, version_codes
@@ -152,8 +152,9 @@ async def resolve_spec_release_from_db( 
                return await _auto_crawl_and_resolve(

                    normalized_spec,

                    requested_release,

                    db_path,

                    db_file,

                    db=db,

                    db_path=db_path,

                    db_file=db_file,

                )

    except (OSError, ValueError) as exc:

        _logger.debug(
@@ -169,8 +170,9 @@ async def resolve_spec_release_from_db( 
async def _auto_crawl_and_resolve(

    normalized_spec: str,

    requested_release: str,

    db_path: Path | None,

    db_file: Path | None,

    db: SpecDatabase | None = None,

    db_path: Path | None = None,

    db_file: Path | None = None,

) -> tuple[str, list[str]]:

    """Crawl spec metadata from 3GPP, store versions, then retry resolution."""

    try:
@@ -178,8 +180,11 @@ async def _auto_crawl_and_resolve( 
        effective_db = db_path if db_path is not None else PathConfig().db_file

        if effective_db is None:

            return requested_release, []

        async with SpecDatabase(effective_db) as db:

        if db is not None:

            await db.crawl_specs([normalized_spec], requested_release, sources)

        else:

            async with SpecDatabase(effective_db) as inner_db:

                await inner_db.crawl_specs([normalized_spec], requested_release, sources)

        return await resolve_spec_release_from_db(

            normalized_spec,

            requested_release,