diff --git a/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py b/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py index 071f7286e6..ca3cdc9f47 100644 --- a/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py +++ b/airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py @@ -19,7 +19,7 @@ GTFSDownloadConfigExtract, get_fs, ) -from pydantic import ValidationError +from pydantic.v1 import ValidationError def gtfs_datasets_to_extract_configs( diff --git a/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py b/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py index 38aad85c8e..dde36ccc17 100644 --- a/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py +++ b/airflow/dags/download_gtfs_schedule_v2/download_schedule_feeds.py @@ -24,7 +24,7 @@ get_fs, get_latest, ) -from pydantic import validator +from pydantic.v1 import validator from requests.exceptions import HTTPError GTFS_FEED_LIST_ERROR_THRESHOLD = 0.95 diff --git a/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py b/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py index 9aea203828..a97057e5ed 100644 --- a/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py +++ b/airflow/dags/sync_ntd_data_xlsx/scrape_ntd_xlsx_urls.py @@ -6,7 +6,7 @@ import requests from bs4 import BeautifulSoup -from pydantic import HttpUrl, ValidationError, parse_obj_as +from pydantic.v1 import HttpUrl, ValidationError, parse_obj_as from airflow.exceptions import AirflowException diff --git a/airflow/plugins/operators/littlepay_raw_sync.py b/airflow/plugins/operators/littlepay_raw_sync.py index b017d4ff66..2fd64070aa 100644 --- a/airflow/plugins/operators/littlepay_raw_sync.py +++ b/airflow/plugins/operators/littlepay_raw_sync.py @@ -14,9 +14,9 @@ get_fs, get_latest_file, ) -from pydantic.class_validators import validator -from pydantic.error_wrappers import ValidationError -from pydantic.main import BaseModel +from pydantic.v1.class_validators import validator +from pydantic.v1.error_wrappers import ValidationError +from pydantic.v1.main import BaseModel from airflow.models import BaseOperator diff --git a/airflow/plugins/operators/scrape_ntd_xlsx.py b/airflow/plugins/operators/scrape_ntd_xlsx.py index 0d980209a9..15d012cd78 100644 --- a/airflow/plugins/operators/scrape_ntd_xlsx.py +++ b/airflow/plugins/operators/scrape_ntd_xlsx.py @@ -12,7 +12,7 @@ get_fs, make_name_bq_safe, ) -from pydantic import HttpUrl, parse_obj_as +from pydantic.v1 import HttpUrl, parse_obj_as from airflow.exceptions import AirflowException from airflow.models import BaseOperator # type: ignore diff --git a/airflow/plugins/operators/scrape_state_geoportal.py b/airflow/plugins/operators/scrape_state_geoportal.py index 7ebbc389fe..67bd595738 100644 --- a/airflow/plugins/operators/scrape_state_geoportal.py +++ b/airflow/plugins/operators/scrape_state_geoportal.py @@ -7,7 +7,7 @@ import pendulum import requests from calitp_data_infra.storage import PartitionedGCSArtifact, get_fs # type: ignore -from pydantic import HttpUrl, parse_obj_as +from pydantic.v1 import HttpUrl, parse_obj_as from airflow.exceptions import AirflowException from airflow.models import BaseOperator # type: ignore diff --git a/airflow/plugins/utils.py b/airflow/plugins/utils.py index b37ab87f24..94e29651f3 100644 --- a/airflow/plugins/utils.py +++ b/airflow/plugins/utils.py @@ -11,7 +11,7 @@ PartitionedGCSArtifact, fetch_all_in_partition, ) -from pydantic import validator +from pydantic.v1 import validator SCHEDULE_UNZIPPED_BUCKET = os.environ["CALITP_BUCKET__GTFS_SCHEDULE_UNZIPPED"] SCHEDULE_UNZIPPED_BUCKET_HOURLY = os.environ[ diff --git a/airflow/requirements.txt b/airflow/requirements.txt index b2351c3442..9ca496fd5b 100644 --- a/airflow/requirements.txt +++ b/airflow/requirements.txt @@ -1,11 +1,11 @@ calitp-data-infra==2025.6.5 gusty==0.6.0 pyairtable==2.2.1 -pydantic>=1.9,<2.0 +pydantic>=1.9 # 2.10.5 for latest composer typer==0.4.1 sentry-sdk==1.17.0 -platformdirs>=2.5 -boto3>=1.26.87,<2 +platformdirs>=2.5 # 4.3.6 for latest composer +boto3>=1.26.87 # 1.36.15 for latest composer openpyxl==3.1.5 beautifulsoup4==4.12.3 astronomer-cosmos diff --git a/packages/calitp-data-infra/calitp_data_infra/storage.py b/packages/calitp-data-infra/calitp_data_infra/storage.py index faf9945c1b..738cc0c549 100644 --- a/packages/calitp-data-infra/calitp_data_infra/storage.py +++ b/packages/calitp-data-infra/calitp_data_infra/storage.py @@ -33,7 +33,7 @@ import humanize import pendulum from google.cloud import storage # type: ignore -from pydantic import ( +from pydantic.v1 import ( BaseModel, ConstrainedStr, Extra, @@ -42,8 +42,8 @@ ValidationError, validator, ) -from pydantic.class_validators import root_validator -from pydantic.tools import parse_obj_as +from pydantic.v1.class_validators import root_validator +from pydantic.v1.tools import parse_obj_as from requests import Request, Session from typing_extensions import Annotated, Literal diff --git a/packages/calitp-data-infra/pyproject.toml b/packages/calitp-data-infra/pyproject.toml index 6f1e8e36d0..71330934f1 100644 --- a/packages/calitp-data-infra/pyproject.toml +++ b/packages/calitp-data-infra/pyproject.toml @@ -8,12 +8,13 @@ authors = ["Andrew Vaccaro"] [tool.poetry.dependencies] python = ">=3.8,<3.12" # Some of these are pinned oddly to play nicely with Composer -pydantic = ">=1.9,<2.0" +pydantic = ">1.9" tqdm = "^4.64.1" -pendulum = "^2.1.2" +pendulum = ">2.1.2" humanize = "^4.6.0" backoff = "^2.2.1" typing-extensions = ">=3.10.0.2" +google-cloud-secret-manager = ">2.16.4" gcsfs = "!=2022.7.1" google-api-core = "^2.24.1" google-cloud-secret-manager = "^2.23.1" diff --git a/packages/calitp-data-infra/tests/test_storage.py b/packages/calitp-data-infra/tests/test_storage.py index e2b669c87b..0e51139a95 100644 --- a/packages/calitp-data-infra/tests/test_storage.py +++ b/packages/calitp-data-infra/tests/test_storage.py @@ -1,7 +1,7 @@ import pendulum import pytest from calitp_data_infra.storage import GTFSDownloadConfig, make_name_bq_safe -from pydantic import ValidationError +from pydantic.v1 import ValidationError def test_gtfs_download_config() -> None: