Skip to content

Upgrade to pydantic 2.0 #3680

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Upgrade to pydantic 2.0 #3680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2383ce2
Update calitp-data-infra package dependencies for latest Cloud Compos…
mjumbewu Feb 7, 2025
7d6b466
Update airflow requirements for latest cloud composer
mjumbewu Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
GTFSDownloadConfigExtract,
get_fs,
)
from pydantic import ValidationError
from pydantic.v1 import ValidationError


def gtfs_datasets_to_extract_configs(
Expand Down
2 changes: 1 addition & 1 deletion airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
get_fs,
get_latest,
)
from pydantic import validator
from pydantic.v1 import validator
from requests.exceptions import HTTPError

GTFS_FEED_LIST_ERROR_THRESHOLD = 0.95
Expand Down
2 changes: 1 addition & 1 deletion airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import requests
from bs4 import BeautifulSoup
from pydantic import HttpUrl, ValidationError, parse_obj_as
from pydantic.v1 import HttpUrl, ValidationError, parse_obj_as

from airflow.exceptions import AirflowException

Expand Down
6 changes: 3 additions & 3 deletions airflow/plugins/operators/littlepay_raw_sync.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
get_fs,
get_latest_file,
)
from pydantic.class_validators import validator
from pydantic.error_wrappers import ValidationError
from pydantic.main import BaseModel
from pydantic.v1.class_validators import validator
from pydantic.v1.error_wrappers import ValidationError
from pydantic.v1.main import BaseModel

from airflow.models import BaseOperator

Expand Down
2 changes: 1 addition & 1 deletion airflow/plugins/operators/scrape_ntd_xlsx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
get_fs,
make_name_bq_safe,
)
from pydantic import HttpUrl, parse_obj_as
from pydantic.v1 import HttpUrl, parse_obj_as

from airflow.exceptions import AirflowException
from airflow.models import BaseOperator # type: ignore
Expand Down
2 changes: 1 addition & 1 deletion airflow/plugins/operators/scrape_state_geoportal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import pendulum
import requests
from calitp_data_infra.storage import PartitionedGCSArtifact, get_fs # type: ignore
from pydantic import HttpUrl, parse_obj_as
from pydantic.v1 import HttpUrl, parse_obj_as

from airflow.exceptions import AirflowException
from airflow.models import BaseOperator # type: ignore
Expand Down
2 changes: 1 addition & 1 deletion airflow/plugins/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
PartitionedGCSArtifact,
fetch_all_in_partition,
)
from pydantic import validator
from pydantic.v1 import validator

SCHEDULE_UNZIPPED_BUCKET = os.environ["CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED"]
SCHEDULE_UNZIPPED_BUCKET_HOURLY = os.environ[
Expand Down
6 changes: 3 additions & 3 deletions airflow/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
calitp-data-infra==2025.6.5
gusty==0.6.0
pyairtable==2.2.1
pydantic>=1.9,<2.0
pydantic>=1.9 # 2.10.5 for latest composer
typer==0.4.1
sentry-sdk==1.17.0
platformdirs>=2.5
boto3>=1.26.87,<2
platformdirs>=2.5 # 4.3.6 for latest composer
boto3>=1.26.87 # 1.36.15 for latest composer
openpyxl==3.1.5
beautifulsoup4==4.12.3
astronomer-cosmos
6 changes: 3 additions & 3 deletions packages/calitp-data-infra/calitp_data_infra/storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import humanize
import pendulum
from google.cloud import storage # type: ignore
from pydantic import (
from pydantic.v1 import (
BaseModel,
ConstrainedStr,
Extra,
Expand All @@ -42,8 +42,8 @@
ValidationError,
validator,
)
from pydantic.class_validators import root_validator
from pydantic.tools import parse_obj_as
from pydantic.v1.class_validators import root_validator
from pydantic.v1.tools import parse_obj_as
from requests import Request, Session
from typing_extensions import Annotated, Literal

Expand Down
5 changes: 3 additions & 2 deletions packages/calitp-data-infra/pyproject.toml
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend moving these changes to a separate PR so it gets created and uploaded first, and you can use the package in the rest of the PR.

Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ authors = ["Andrew Vaccaro"]
[tool.poetry.dependencies]
python = ">=3.8,<3.12"
# Some of these are pinned oddly to play nicely with Composer
pydantic = ">=1.9,<2.0"
pydantic = ">1.9"
tqdm = "^4.64.1"
pendulum = "^2.1.2"
pendulum = ">2.1.2"
humanize = "^4.6.0"
backoff = "^2.2.1"
typing-extensions = ">=3.10.0.2"
google-cloud-secret-manager = ">2.16.4"
gcsfs = "!=2022.7.1"
google-api-core = "^2.24.1"
google-cloud-secret-manager = "^2.23.1"
Expand Down
2 changes: 1 addition & 1 deletion packages/calitp-data-infra/tests/test_storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pendulum
import pytest
from calitp_data_infra.storage import GTFSDownloadConfig, make_name_bq_safe
from pydantic import ValidationError
from pydantic.v1 import ValidationError


def test_gtfs_download_config() -> None:
Expand Down
Loading